2024-04-10T02:45:45.542 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T02:45:45.543 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi --archive /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943 --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --owner scheduled_yuriw@teuthology -- /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/orig.config.yaml 2024-04-10T02:45:45.603 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T02:45:45.708 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943 branch: wip-yuri4-testing-2024-04-08-1432 description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: yweinste@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7648943' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri4-testing-2024-04-08-1432 ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-yuri4-testing-2024-04-08-1432 sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e owner: scheduled_yuriw@teuthology priority: 99 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 7233 sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 111/120000 suite: upgrade suite_branch: wip-yuri4-testing-2024-04-08-1432 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e targets: smithi102.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKUOtJ+0rPgrgh8t0qmo1iUjWt0JrofPmzTh2tK8FTK0nHjSmLQrr5JRPfFEVmtK4CLHgfwnkwRbAIQ258cbKG0= smithi152.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBPIC4a+x5o0awOAOn+29xG+SC+y6GFDZ1FYsJodODlZJFj9O5AwKCkH9eJ7xfk4W0vL2Hfg9QBu+pL6gx5gttg= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-09_14:58:21 tube: smithi user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T02:45:45.708 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa; will attempt to use it 2024-04-10T02:45:45.709 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa/tasks 2024-04-10T02:45:45.709 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T02:45:45.711 INFO:teuthology.task.internal:Checking packages... 2024-04-10T02:45:45.734 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e' 2024-04-10T02:45:45.735 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:45:45.735 INFO:teuthology.packaging:ref: None 2024-04-10T02:45:45.735 INFO:teuthology.packaging:tag: None 2024-04-10T02:45:45.735 INFO:teuthology.packaging:branch: wip-yuri4-testing-2024-04-08-1432 2024-04-10T02:45:45.735 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:45:45.735 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri4-testing-2024-04-08-1432 2024-04-10T02:45:45.951 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2051.g3a866975 2024-04-10T02:45:45.952 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T02:45:45.959 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T02:45:45.959 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T02:45:46.057 INFO:teuthology.task.internal:Saving configuration 2024-04-10T02:45:46.080 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T02:45:46.087 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T02:45:46.113 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi102.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 02:40:37.713075', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKUOtJ+0rPgrgh8t0qmo1iUjWt0JrofPmzTh2tK8FTK0nHjSmLQrr5JRPfFEVmtK4CLHgfwnkwRbAIQ258cbKG0='} 2024-04-10T02:45:46.136 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 02:40:37.714986', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBPIC4a+x5o0awOAOn+29xG+SC+y6GFDZ1FYsJodODlZJFj9O5AwKCkH9eJ7xfk4W0vL2Hfg9QBu+pL6gx5gttg='} 2024-04-10T02:45:46.136 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T02:45:46.143 INFO:teuthology.task.internal:roles: ubuntu@smithi102.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T02:45:46.143 INFO:teuthology.task.internal:roles: ubuntu@smithi152.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T02:45:46.143 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T02:45:46.281 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb9502e7160>, signals=[15]) 2024-04-10T02:45:46.281 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T02:45:46.287 INFO:teuthology.task.internal:Opening connections... 2024-04-10T02:45:46.288 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T02:45:46.290 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:45:46.377 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi152.front.sepia.ceph.com 2024-04-10T02:45:46.378 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:45:46.470 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T02:45:46.477 DEBUG:teuthology.orchestra.run.smithi102:> uname -m 2024-04-10T02:45:46.495 INFO:teuthology.orchestra.run.smithi102.stdout:x86_64 2024-04-10T02:45:46.495 DEBUG:teuthology.orchestra.run.smithi102:> cat /etc/os-release 2024-04-10T02:45:46.552 INFO:teuthology.orchestra.run.smithi102.stdout:NAME="CentOS Stream" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION="9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:ID="centos" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:ID_LIKE="rhel fedora" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION_ID="9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:PLATFORM_ID="platform:el9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:ANSI_COLOR="0;31" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:LOGO="fedora-logo-icon" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:HOME_URL="https://centos.org/" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-10T02:45:46.553 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T02:45:46.554 INFO:teuthology.lock.ops:Updating smithi102.front.sepia.ceph.com on lock server 2024-04-10T02:45:46.583 DEBUG:teuthology.orchestra.run.smithi152:> uname -m 2024-04-10T02:45:46.601 INFO:teuthology.orchestra.run.smithi152.stdout:x86_64 2024-04-10T02:45:46.601 DEBUG:teuthology.orchestra.run.smithi152:> cat /etc/os-release 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:NAME="CentOS Stream" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION="9" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:ID="centos" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:ID_LIKE="rhel fedora" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION_ID="9" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:PLATFORM_ID="platform:el9" 2024-04-10T02:45:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:ANSI_COLOR="0;31" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:LOGO="fedora-logo-icon" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:HOME_URL="https://centos.org/" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-10T02:45:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T02:45:46.658 INFO:teuthology.lock.ops:Updating smithi152.front.sepia.ceph.com on lock server 2024-04-10T02:45:46.679 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T02:45:46.689 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T02:45:46.758 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T02:45:46.758 DEBUG:teuthology.orchestra.run.smithi102:> test '!' -e /home/ubuntu/cephtest 2024-04-10T02:45:46.760 DEBUG:teuthology.orchestra.run.smithi152:> test '!' -e /home/ubuntu/cephtest 2024-04-10T02:45:46.776 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T02:45:46.810 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T02:45:46.810 DEBUG:teuthology.orchestra.run.smithi102:> test -z $(ls -A /var/lib/ceph) 2024-04-10T02:45:46.817 DEBUG:teuthology.orchestra.run.smithi152:> test -z $(ls -A /var/lib/ceph) 2024-04-10T02:45:46.849 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T02:45:46.904 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T02:45:46.938 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T02:45:46.939 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T02:45:46.939 DEBUG:teuthology.orchestra.run.smithi102:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T02:45:46.939 DEBUG:teuthology.orchestra.run.smithi152:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T02:45:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T02:45:46.955 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-04-10T02:45:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T02:45:46.956 DEBUG:teuthology.orchestra.run.smithi102:> uname -r 2024-04-10T02:45:47.011 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-361.el9.x86_64 2024-04-10T02:45:47.012 INFO:teuthology.task.kernel:Running kernel on smithi152: 5.14.0-361.el9.x86_64 2024-04-10T02:45:47.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum install -y kernel 2024-04-10T02:45:47.013 INFO:teuthology.orchestra.run.smithi102.stdout:5.14.0-361.el9.x86_64 2024-04-10T02:45:47.013 INFO:teuthology.task.kernel:Running kernel on smithi102: 5.14.0-361.el9.x86_64 2024-04-10T02:45:47.013 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum install -y kernel 2024-04-10T02:45:49.169 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - AppStream 14 MB/s | 8.1 MB 00:00 2024-04-10T02:45:49.192 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS-9 - AppStream 15 MB/s | 8.1 MB 00:00 2024-04-10T02:45:50.850 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS-9 - BaseOS 7.4 MB/s | 2.5 MB 00:00 2024-04-10T02:45:50.936 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - BaseOS 5.5 MB/s | 2.5 MB 00:00 2024-04-10T02:45:51.782 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.0 MB 00:00 2024-04-10T02:45:51.885 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.0 MB 00:00 2024-04-10T02:45:54.600 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-10T02:45:58.609 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.2 MB 00:00 2024-04-10T02:46:00.068 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 16 kB 00:00 2024-04-10T02:46:02.189 INFO:teuthology.orchestra.run.smithi102.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-04-10T02:46:07.312 INFO:teuthology.orchestra.run.smithi102.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:46:09.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:46:09.546 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:46:09.547 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.548 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.549 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.550 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.551 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.552 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.553 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:46:09.944 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-10T02:46:09.944 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-10T02:46:10.055 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:46:10.067 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:46:10.068 INFO:teuthology.orchestra.run.smithi102.stdout:Install 4 Packages 2024-04-10T02:46:10.068 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:46:10.068 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 77 M 2024-04-10T02:46:10.068 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 123 M 2024-04-10T02:46:10.068 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:46:10.471 INFO:teuthology.orchestra.run.smithi102.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 619 kB/s | 214 kB 00:00 2024-04-10T02:46:15.484 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 860 kB/s | 19 MB 00:22 2024-04-10T02:46:17.196 INFO:teuthology.orchestra.run.smithi102.stdout:(2/4): kernel-modules-core-5.14.0-435.el9.x86_6 4.1 MB/s | 28 MB 00:06 2024-04-10T02:46:17.346 INFO:teuthology.orchestra.run.smithi102.stdout:(3/4): kernel-core-5.14.0-435.el9.x86_64.rpm 2.1 MB/s | 15 MB 00:07 2024-04-10T02:46:21.046 INFO:teuthology.orchestra.run.smithi102.stdout:(4/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 3.1 MB/s | 34 MB 00:10 2024-04-10T02:46:21.046 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:46:21.046 INFO:teuthology.orchestra.run.smithi102.stdout:Total 7.0 MB/s | 77 MB 00:10 2024-04-10T02:46:21.719 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:46:22.094 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:46:22.094 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:46:23.115 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:46:23.116 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:46:25.677 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:46:26.307 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-10T02:46:26.397 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:46:28.408 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 684 kB/s | 6.2 MB 00:09 2024-04-10T02:46:29.204 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:46:29.343 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:46:30.092 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 31 kB/s | 16 kB 00:00 2024-04-10T02:46:32.387 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 9.7 MB/s | 21 MB 00:02 2024-04-10T02:46:34.041 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:46:34.078 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:46:37.493 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-10T02:46:38.782 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:46:39.730 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.731 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.732 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:46:39.733 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.734 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.735 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.736 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.737 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:46:39.738 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.738 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.738 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:46:39.738 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:46:40.151 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-10T02:46:40.151 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-10T02:46:40.272 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:46:40.283 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout:Install 4 Packages 2024-04-10T02:46:40.284 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:46:40.285 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 77 M 2024-04-10T02:46:40.285 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 123 M 2024-04-10T02:46:40.285 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:46:40.561 INFO:teuthology.orchestra.run.smithi152.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 886 kB/s | 214 kB 00:00 2024-04-10T02:46:41.828 INFO:teuthology.orchestra.run.smithi152.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 10 MB/s | 15 MB 00:01 2024-04-10T02:46:42.276 INFO:teuthology.orchestra.run.smithi152.stdout:(3/4): kernel-modules-core-5.14.0-435.el9.x86_6 16 MB/s | 28 MB 00:01 2024-04-10T02:46:42.737 INFO:teuthology.orchestra.run.smithi152.stdout:(4/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 14 MB/s | 34 MB 00:02 2024-04-10T02:46:42.737 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:46:42.737 INFO:teuthology.orchestra.run.smithi152.stdout:Total 31 MB/s | 77 MB 00:02 2024-04-10T02:46:43.410 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:46:43.791 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:46:43.791 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:46:44.841 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:46:44.842 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:46:47.329 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:46:48.002 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-10T02:46:48.099 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:46:50.887 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:46:51.029 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:46:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:46:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:00.542 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:07.850 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:07.850 INFO:teuthology.orchestra.run.smithi102.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-10T02:47:07.850 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:47:07.896 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:08.209 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:08.209 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-10T02:47:08.209 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:47:08.209 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-10T02:47:08.663 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-10T02:47:08.664 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-10T02:47:08.664 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:47:08.664 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:47:09.006 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T02:47:09.108 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:09.108 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-10T02:47:09.109 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-10T02:47:09.109 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T02:47:09.109 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T02:47:09.109 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T02:47:09.109 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi102.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T02:47:09.109 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T02:47:09.190 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:09.190 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-10T02:47:09.190 DEBUG:teuthology.orchestra.run.smithi102:> sudo rpm -qi grub2-tools 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Name : grub2-tools 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Epoch : 1 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Version : 2.06 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Release : 68.el9 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Architecture: x86_64 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Group : Unspecified 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:Size : 8264254 2024-04-10T02:47:09.233 INFO:teuthology.orchestra.run.smithi102.stdout:License : GPLv3+ 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Packager : builder@centos.org 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Vendor : CentOS 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Summary : Support tools for GRUB. 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:Description : 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T02:47:09.234 INFO:teuthology.orchestra.run.smithi102.stdout:hardware devices. 2024-04-10T02:47:09.235 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:47:09.235 INFO:teuthology.orchestra.run.smithi102.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T02:47:09.235 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T02:47:09.235 INFO:teuthology.task.kernel:Updating grub on smithi102 to boot 5.14.0-435.el9.x86_64 2024-04-10T02:47:09.236 DEBUG:teuthology.orchestra.run.smithi102:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T02:47:09.838 INFO:teuthology.orchestra.run.smithi102.stderr:Generating grub configuration file ... 2024-04-10T02:47:11.202 INFO:teuthology.orchestra.run.smithi102.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-10T02:47:11.230 INFO:teuthology.orchestra.run.smithi102.stderr:done 2024-04-10T02:47:11.232 DEBUG:teuthology.orchestra.run.smithi102:> mktemp 2024-04-10T02:47:11.247 INFO:teuthology.orchestra.run.smithi102.stdout:/tmp/tmp.BmfKf8U1NC 2024-04-10T02:47:11.247 DEBUG:teuthology.orchestra.run.smithi102:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.BmfKf8U1NC 2024-04-10T02:47:11.316 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 0666 /tmp/tmp.BmfKf8U1NC 2024-04-10T02:47:11.451 DEBUG:teuthology.orchestra.remote:smithi102:/tmp/tmp.BmfKf8U1NC is 6KB 2024-04-10T02:47:11.502 DEBUG:teuthology.orchestra.run.smithi102:> rm -fr /tmp/tmp.BmfKf8U1NC 2024-04-10T02:47:11.516 DEBUG:teuthology.orchestra.run.smithi102:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T02:47:11.592 INFO:teuthology.orchestra.run.smithi102.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-10T02:47:11.592 INFO:teuthology.orchestra.run.smithi102.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-10T02:47:11.592 INFO:teuthology.orchestra.run.smithi102.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-10T02:47:11.593 INFO:teuthology.orchestra.run.smithi102.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-10T02:47:11.593 DEBUG:teuthology.orchestra.run.smithi102:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-10T02:47:11.751 DEBUG:teuthology.orchestra.run.smithi102:> sudo shutdown -r now 2024-04-10T02:47:29.944 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:29.944 INFO:teuthology.orchestra.run.smithi152.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-10T02:47:29.944 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:47:29.989 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:30.430 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:30.430 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-10T02:47:30.430 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-10T02:47:30.431 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-10T02:47:30.848 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-10T02:47:30.849 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:47:30.849 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:47:30.969 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:30.969 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-10T02:47:30.970 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-10T02:47:30.970 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-10T02:47:30.970 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:47:30.970 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:47:31.188 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T02:47:31.298 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:31.298 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-10T02:47:31.298 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-10T02:47:31.298 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T02:47:31.298 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T02:47:31.298 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T02:47:31.299 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi152.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T02:47:31.299 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T02:47:31.379 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-10T02:47:31.380 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-10T02:47:31.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo rpm -qi grub2-tools 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Name : grub2-tools 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Epoch : 1 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Version : 2.06 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Release : 68.el9 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Architecture: x86_64 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Group : Unspecified 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Size : 8264254 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:License : GPLv3+ 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-10T02:47:31.418 INFO:teuthology.orchestra.run.smithi152.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Packager : builder@centos.org 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Vendor : CentOS 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Summary : Support tools for GRUB. 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:Description : 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:hardware devices. 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:47:31.419 INFO:teuthology.orchestra.run.smithi152.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T02:47:31.420 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T02:47:31.420 INFO:teuthology.task.kernel:Updating grub on smithi152 to boot 5.14.0-435.el9.x86_64 2024-04-10T02:47:31.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T02:47:32.072 INFO:teuthology.orchestra.run.smithi152.stderr:Generating grub configuration file ... 2024-04-10T02:47:33.492 INFO:teuthology.orchestra.run.smithi152.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-10T02:47:33.523 INFO:teuthology.orchestra.run.smithi152.stderr:done 2024-04-10T02:47:33.525 DEBUG:teuthology.orchestra.run.smithi152:> mktemp 2024-04-10T02:47:33.581 INFO:teuthology.orchestra.run.smithi152.stdout:/tmp/tmp.grGZXYa3vI 2024-04-10T02:47:33.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.grGZXYa3vI 2024-04-10T02:47:33.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0666 /tmp/tmp.grGZXYa3vI 2024-04-10T02:47:33.784 DEBUG:teuthology.orchestra.remote:smithi152:/tmp/tmp.grGZXYa3vI is 6KB 2024-04-10T02:47:33.832 DEBUG:teuthology.orchestra.run.smithi152:> rm -fr /tmp/tmp.grGZXYa3vI 2024-04-10T02:47:33.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T02:47:33.913 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-10T02:47:33.913 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-10T02:47:33.913 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-10T02:47:33.913 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-10T02:47:33.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-10T02:47:34.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo shutdown -r now 2024-04-10T02:47:41.758 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T02:47:41.759 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T02:47:41.759 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:48:04.089 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T02:48:04.090 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:48:04.091 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:48:38.687 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-04-10T02:48:41.765 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T02:48:47.690 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:48:47.691 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:48:50.768 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T02:48:50.769 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:48:54.285 DEBUG:teuthology.orchestra.run.smithi102:> true 2024-04-10T02:48:54.714 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T02:48:54.715 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-435.el9.x86_64"... 2024-04-10T02:48:54.715 DEBUG:teuthology.orchestra.run.smithi102:> uname -r 2024-04-10T02:48:54.729 INFO:teuthology.orchestra.run.smithi102.stdout:5.14.0-435.el9.x86_64 2024-04-10T02:48:54.729 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-10T02:48:54.729 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T02:48:54.729 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T02:48:55.730 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T02:48:55.730 DEBUG:teuthology.orchestra.run.smithi102:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T02:48:55.798 INFO:teuthology.orchestra.run.smithi102.stdout:ttyS1 2024-04-10T02:48:55.844 DEBUG:teuthology.parallel:result is None 2024-04-10T02:49:06.082 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-04-10T02:49:18.083 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:49:18.084 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:49:20.128 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-04-10T02:49:20.609 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:49:20.609 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-435.el9.x86_64"... 2024-04-10T02:49:20.609 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-04-10T02:49:20.623 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-435.el9.x86_64 2024-04-10T02:49:20.623 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-10T02:49:20.623 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T02:49:20.623 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T02:49:21.624 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T02:49:21.624 DEBUG:teuthology.orchestra.run.smithi152:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T02:49:21.711 INFO:teuthology.orchestra.run.smithi152.stdout:ttyS1 2024-04-10T02:49:21.754 DEBUG:teuthology.parallel:result is None 2024-04-10T02:49:21.754 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T02:49:21.763 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T02:49:21.763 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T02:49:21.766 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T02:49:21.783 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T02:49:21.791 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T02:49:21.797 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T02:49:21.797 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T02:49:21.826 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T02:49:21.875 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T02:49:21.882 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T02:49:21.882 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T02:49:21.885 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T02:49:21.915 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T02:49:21.917 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T02:49:21.929 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T02:49:21.930 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T02:49:21.931 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T02:49:21.938 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T02:49:21.938 DEBUG:teuthology.orchestra.run.smithi102:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T02:49:21.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T02:49:22.001 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T02:49:22.011 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T02:49:22.011 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T02:49:22.044 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T02:49:22.058 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T02:49:22.140 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T02:49:22.197 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T02:49:22.281 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T02:49:22.321 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:49:22.321 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T02:49:22.388 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T02:49:22.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T02:49:22.479 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T02:49:22.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T02:49:22.605 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:49:22.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T02:49:22.674 DEBUG:teuthology.orchestra.run.smithi102:> sudo service rsyslog restart 2024-04-10T02:49:22.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo service rsyslog restart 2024-04-10T02:49:22.728 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T02:49:22.763 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T02:49:23.081 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T02:49:23.089 INFO:teuthology.task.internal:Starting timer... 2024-04-10T02:49:23.089 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T02:49:23.099 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T02:49:23.106 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T02:49:23.107 DEBUG:teuthology.orchestra.run.smithi102:> sudo service auditd rotate 2024-04-10T02:49:23.170 INFO:teuthology.orchestra.run.smithi102.stdout:Rotating logs: 2024-04-10T02:49:23.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo service auditd rotate 2024-04-10T02:49:23.220 INFO:teuthology.orchestra.run.smithi152.stdout:Rotating logs: 2024-04-10T02:49:23.222 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T02:49:23.222 DEBUG:teuthology.orchestra.run.smithi102:> /usr/sbin/getenforce 2024-04-10T02:49:23.249 INFO:teuthology.orchestra.run.smithi102.stdout:Permissive 2024-04-10T02:49:23.249 DEBUG:teuthology.orchestra.run.smithi152:> /usr/sbin/getenforce 2024-04-10T02:49:23.265 INFO:teuthology.orchestra.run.smithi152.stdout:Permissive 2024-04-10T02:49:23.265 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi102.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi152.front.sepia.ceph.com': 'permissive'} 2024-04-10T02:49:23.265 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-10T02:49:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T02:49:23.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-10T02:49:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T02:49:23.345 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T02:49:23.346 DEBUG:teuthology.orchestra.run.smithi102:> sudo /usr/sbin/setenforce permissive 2024-04-10T02:49:23.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo /usr/sbin/setenforce permissive 2024-04-10T02:49:23.435 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T02:49:23.445 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T02:49:23.451 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T02:49:23.575 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T02:49:23.591 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T02:49:23.592 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi102.front.sepia.ceph.com,smithi152.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T02:53:47.807 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi102.front.sepia.ceph.com'), Remote(name='ubuntu@smithi152.front.sepia.ceph.com')] 2024-04-10T02:53:47.808 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T02:53:47.809 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:53:47.898 DEBUG:teuthology.orchestra.run.smithi102:> true 2024-04-10T02:53:47.938 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T02:53:47.941 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:53:47.942 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T02:53:48.027 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-04-10T02:53:48.105 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-10T02:53:48.105 INFO:teuthology.run_tasks:Running task clock... 2024-04-10T02:53:48.118 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-10T02:53:48.118 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-10T02:53:48.118 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-10T02:53:48.121 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-10T02:53:48.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-10T02:53:48.150 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-10T02:53:48.166 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-10T02:53:48.176 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-10T02:53:48.193 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-10T02:53:48.195 INFO:teuthology.orchestra.run.smithi102.stderr:sudo: ntpd: command not found 2024-04-10T02:53:48.209 INFO:teuthology.orchestra.run.smithi102.stdout:506 Cannot talk to daemon 2024-04-10T02:53:48.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: ntpd: command not found 2024-04-10T02:53:48.226 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-10T02:53:48.236 INFO:teuthology.orchestra.run.smithi152.stdout:506 Cannot talk to daemon 2024-04-10T02:53:48.242 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-10T02:53:48.252 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-10T02:53:48.268 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-10T02:53:48.301 INFO:teuthology.orchestra.run.smithi102.stderr:bash: line 1: ntpq: command not found 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:=============================================================================== 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.305 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.322 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.325 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T02:53:48.326 INFO:teuthology.run_tasks:Running task install... 2024-04-10T02:53:48.335 DEBUG:teuthology.task.install:project ceph 2024-04-10T02:53:48.335 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e'}} 2024-04-10T02:53:48.335 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e'} 2024-04-10T02:53:48.335 INFO:teuthology.task.install:Using flavor: default 2024-04-10T02:53:48.341 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-10T02:53:48.341 INFO:teuthology.task.install:extra packages: [] 2024-04-10T02:53:48.341 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e', 'tag': None, 'wait_for_package': False} 2024-04-10T02:53:48.342 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:53:48.342 INFO:teuthology.packaging:ref: None 2024-04-10T02:53:48.342 INFO:teuthology.packaging:tag: None 2024-04-10T02:53:48.342 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:53:48.342 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:53:48.342 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-10T02:53:48.344 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e', 'tag': None, 'wait_for_package': False} 2024-04-10T02:53:48.344 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:53:48.344 INFO:teuthology.packaging:ref: None 2024-04-10T02:53:48.344 INFO:teuthology.packaging:tag: None 2024-04-10T02:53:48.344 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:53:48.344 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:53:48.345 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-10T02:53:48.566 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/ 2024-04-10T02:53:48.567 INFO:teuthology.task.install.rpm:Package version is 17.2.7-825.g896bb95d 2024-04-10T02:53:48.668 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/ 2024-04-10T02:53:48.668 INFO:teuthology.task.install.rpm:Package version is 17.2.7-825.g896bb95d 2024-04-10T02:53:48.674 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-10T02:53:48.674 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:53:48.674 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-10T02:53:48.702 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-10T02:53:48.703 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:53:48.703 INFO:teuthology.packaging:ref: None 2024-04-10T02:53:48.703 INFO:teuthology.packaging:tag: None 2024-04-10T02:53:48.703 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:53:48.703 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:53:48.703 DEBUG:teuthology.orchestra.run.smithi102:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-10T02:53:48.771 DEBUG:teuthology.orchestra.run.smithi102:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-10T02:53:48.835 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/896bb95d082477f698d6cb40968cdae50b9945f3/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-10T02:53:48.835 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:53:48.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-10T02:53:48.851 DEBUG:teuthology.orchestra.run.smithi102:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-10T02:53:48.865 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-10T02:53:48.865 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:53:48.865 INFO:teuthology.packaging:ref: None 2024-04-10T02:53:48.865 INFO:teuthology.packaging:tag: None 2024-04-10T02:53:48.865 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:53:48.865 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:53:48.865 DEBUG:teuthology.orchestra.run.smithi152:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-10T02:53:48.917 INFO:teuthology.orchestra.run.smithi102.stdout:check_obsoletes = 1 2024-04-10T02:53:48.919 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum clean all 2024-04-10T02:53:48.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-10T02:53:49.013 DEBUG:teuthology.orchestra.run.smithi152:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-10T02:53:49.079 INFO:teuthology.orchestra.run.smithi152.stdout:check_obsoletes = 1 2024-04-10T02:53:49.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum clean all 2024-04-10T02:53:49.166 INFO:teuthology.orchestra.run.smithi102.stdout:74 files removed 2024-04-10T02:53:49.193 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-radosgw 2024-04-10T02:53:49.333 INFO:teuthology.orchestra.run.smithi152.stdout:74 files removed 2024-04-10T02:53:49.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-radosgw 2024-04-10T02:53:49.926 INFO:teuthology.orchestra.run.smithi102.stdout:ceph packages for x86_64 179 kB/s | 77 kB 00:00 2024-04-10T02:53:50.124 INFO:teuthology.orchestra.run.smithi152.stdout:ceph packages for x86_64 167 kB/s | 77 kB 00:00 2024-04-10T02:53:50.343 INFO:teuthology.orchestra.run.smithi102.stdout:ceph noarch packages 25 kB/s | 9.2 kB 00:00 2024-04-10T02:53:50.474 INFO:teuthology.orchestra.run.smithi152.stdout:ceph noarch packages 30 kB/s | 9.2 kB 00:00 2024-04-10T02:53:50.677 INFO:teuthology.orchestra.run.smithi102.stdout:ceph source packages 6.0 kB/s | 1.8 kB 00:00 2024-04-10T02:53:50.816 INFO:teuthology.orchestra.run.smithi152.stdout:ceph source packages 5.9 kB/s | 1.8 kB 00:00 2024-04-10T02:53:51.354 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS-9 - AppStream 13 MB/s | 8.1 MB 00:00 2024-04-10T02:53:51.394 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - AppStream 15 MB/s | 8.1 MB 00:00 2024-04-10T02:53:53.295 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - BaseOS 4.3 MB/s | 2.5 MB 00:00 2024-04-10T02:53:53.296 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS-9 - BaseOS 4.1 MB/s | 2.5 MB 00:00 2024-04-10T02:53:54.553 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.0 MB 00:00 2024-04-10T02:53:57.594 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 2.1 MB/s | 8.0 MB 00:03 2024-04-10T02:53:57.663 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - AppStream 9.5 MB/s | 19 MB 00:01 2024-04-10T02:53:59.419 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 26 MB/s | 19 MB 00:00 2024-04-10T02:54:03.072 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - CRB 3.3 MB/s | 6.2 MB 00:01 2024-04-10T02:54:03.362 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.2 MB 00:00 2024-04-10T02:54:04.623 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - Extras packages 77 kB/s | 16 kB 00:00 2024-04-10T02:54:04.712 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 88 kB/s | 16 kB 00:00 2024-04-10T02:54:07.265 INFO:teuthology.orchestra.run.smithi102.stdout:Extra Packages for Enterprise Linux 8.4 MB/s | 21 MB 00:02 2024-04-10T02:54:10.397 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 3.8 MB/s | 21 MB 00:05 2024-04-10T02:54:12.358 INFO:teuthology.orchestra.run.smithi102.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:54:14.966 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.967 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:54:14.968 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:54:14.969 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.970 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.971 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.972 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.973 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:14.974 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:15.430 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-04-10T02:54:15.506 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:54:15.518 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:15.518 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-radosgw x86_64 2:17.2.7-825.g896bb95d.el9 ceph 14 M 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout:Upgrading: 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: librados2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 3.1 M 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: librbd1 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.9 M 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-base x86_64 2:17.2.7-825.g896bb95d.el9 ceph 5.0 M 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-common x86_64 2:17.2.7-825.g896bb95d.el9 ceph 22 M 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-selinux x86_64 2:17.2.7-825.g896bb95d.el9 ceph 25 k 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: libcephfs2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 638 k 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-10T02:54:15.519 INFO:teuthology.orchestra.run.smithi102.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: libradosstriper1 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 457 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: librgw2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 4.9 M 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ceph-argparse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 45 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ceph-common x86_64 2:17.2.7-825.g896bb95d.el9 ceph 118 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cephfs x86_64 2:17.2.7-825.g896bb95d.el9 ceph 156 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rados x86_64 2:17.2.7-825.g896bb95d.el9 ceph 320 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rbd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 292 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rgw x86_64 2:17.2.7-825.g896bb95d.el9 ceph 100 k 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-10T02:54:15.520 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:Install 22 Packages 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:Upgrade 2 Packages 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 58 M 2024-04-10T02:54:15.521 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:54:17.476 INFO:teuthology.orchestra.run.smithi102.stdout:(1/24): ceph-base-17.2.7-825.g896bb95d.el9.x86_ 2.9 MB/s | 5.0 MB 00:01 2024-04-10T02:54:17.684 INFO:teuthology.orchestra.run.smithi102.stdout:(2/24): ceph-common-17.2.7-825.g896bb95d.el9.x8 12 MB/s | 22 MB 00:01 2024-04-10T02:54:17.701 INFO:teuthology.orchestra.run.smithi102.stdout:(3/24): ceph-selinux-17.2.7-825.g896bb95d.el9.x 112 kB/s | 25 kB 00:00 2024-04-10T02:54:17.768 INFO:teuthology.orchestra.run.smithi102.stdout:(4/24): libcephfs2-17.2.7-825.g896bb95d.el9.x86 7.5 MB/s | 638 kB 00:00 2024-04-10T02:54:17.801 INFO:teuthology.orchestra.run.smithi102.stdout:(5/24): libradosstriper1-17.2.7-825.g896bb95d.e 4.5 MB/s | 457 kB 00:00 2024-04-10T02:54:17.877 INFO:teuthology.orchestra.run.smithi102.stdout:(6/24): python3-ceph-argparse-17.2.7-825.g896bb 602 kB/s | 45 kB 00:00 2024-04-10T02:54:17.960 INFO:teuthology.orchestra.run.smithi102.stdout:(7/24): python3-ceph-common-17.2.7-825.g896bb95 1.4 MB/s | 118 kB 00:00 2024-04-10T02:54:17.982 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:54:17.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:54:17.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:54:17.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:54:17.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.989 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.990 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.991 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.992 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:17.993 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:17.994 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.995 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.995 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:17.995 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:18.094 INFO:teuthology.orchestra.run.smithi102.stdout:(8/24): librgw2-17.2.7-825.g896bb95d.el9.x86_64 15 MB/s | 4.9 MB 00:00 2024-04-10T02:54:18.119 INFO:teuthology.orchestra.run.smithi102.stdout:(9/24): python3-cephfs-17.2.7-825.g896bb95d.el9 983 kB/s | 156 kB 00:00 2024-04-10T02:54:18.145 INFO:teuthology.orchestra.run.smithi102.stdout:(10/24): python3-rados-17.2.7-825.g896bb95d.el9 6.2 MB/s | 320 kB 00:00 2024-04-10T02:54:18.186 INFO:teuthology.orchestra.run.smithi102.stdout:(11/24): python3-rgw-17.2.7-825.g896bb95d.el9.x 2.3 MB/s | 100 kB 00:00 2024-04-10T02:54:18.212 INFO:teuthology.orchestra.run.smithi102.stdout:(12/24): python3-rbd-17.2.7-825.g896bb95d.el9.x 3.1 MB/s | 292 kB 00:00 2024-04-10T02:54:18.379 INFO:teuthology.orchestra.run.smithi102.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 272 kB/s | 45 kB 00:00 2024-04-10T02:54:18.527 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:54:18.529 INFO:teuthology.orchestra.run.smithi102.stdout:(14/24): ceph-radosgw-17.2.7-825.g896bb95d.el9. 5.0 MB/s | 14 MB 00:02 2024-04-10T02:54:18.533 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw x86_64 2:17.2.7-825.g896bb95d.el9 ceph 14 M 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: librados2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 3.1 M 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.9 M 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base x86_64 2:17.2.7-825.g896bb95d.el9 ceph 5.0 M 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common x86_64 2:17.2.7-825.g896bb95d.el9 ceph 22 M 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux x86_64 2:17.2.7-825.g896bb95d.el9 ceph 25 k 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-10T02:54:18.534 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 638 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 457 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2 x86_64 2:17.2.7-825.g896bb95d.el9 ceph 4.9 M 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 45 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common x86_64 2:17.2.7-825.g896bb95d.el9 ceph 118 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs x86_64 2:17.2.7-825.g896bb95d.el9 ceph 156 k 2024-04-10T02:54:18.535 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados x86_64 2:17.2.7-825.g896bb95d.el9 ceph 320 k 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 292 k 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw x86_64 2:17.2.7-825.g896bb95d.el9 ceph 100 k 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout:Install 22 Packages 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2024-04-10T02:54:18.536 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:18.537 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 58 M 2024-04-10T02:54:18.537 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:54:18.555 INFO:teuthology.orchestra.run.smithi102.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 436 kB/s | 160 kB 00:00 2024-04-10T02:54:18.613 INFO:teuthology.orchestra.run.smithi102.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 726 kB/s | 42 kB 00:00 2024-04-10T02:54:18.680 INFO:teuthology.orchestra.run.smithi102.stdout:(17/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.1 MB/s | 662 kB 00:00 2024-04-10T02:54:18.805 INFO:teuthology.orchestra.run.smithi102.stdout:(18/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.0 MB/s | 292 kB 00:00 2024-04-10T02:54:18.997 INFO:teuthology.orchestra.run.smithi102.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 167 kB/s | 49 kB 00:00 2024-04-10T02:54:19.081 INFO:teuthology.orchestra.run.smithi102.stdout:(20/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 661 kB/s | 309 kB 00:00 2024-04-10T02:54:19.123 INFO:teuthology.orchestra.run.smithi102.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 212 kB/s | 67 kB 00:00 2024-04-10T02:54:19.323 INFO:teuthology.orchestra.run.smithi102.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 4.9 MB/s | 1.6 MB 00:00 2024-04-10T02:54:19.373 INFO:teuthology.orchestra.run.smithi102.stdout:(23/24): librbd1-17.2.7-825.g896bb95d.el9.x86_6 12 MB/s | 2.9 MB 00:00 2024-04-10T02:54:19.798 INFO:teuthology.orchestra.run.smithi102.stdout:(24/24): librados2-17.2.7-825.g896bb95d.el9.x86 4.3 MB/s | 3.1 MB 00:00 2024-04-10T02:54:19.803 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:19.804 INFO:teuthology.orchestra.run.smithi102.stdout:Total 13 MB/s | 58 MB 00:04 2024-04-10T02:54:19.862 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:54:19.896 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:54:19.896 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:54:20.360 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:54:20.361 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:54:20.381 INFO:teuthology.orchestra.run.smithi152.stdout:(1/24): ceph-base-17.2.7-825.g896bb95d.el9.x86_ 3.0 MB/s | 5.0 MB 00:01 2024-04-10T02:54:20.448 INFO:teuthology.orchestra.run.smithi152.stdout:(2/24): ceph-selinux-17.2.7-825.g896bb95d.el9.x 378 kB/s | 25 kB 00:00 2024-04-10T02:54:20.582 INFO:teuthology.orchestra.run.smithi152.stdout:(3/24): libcephfs2-17.2.7-825.g896bb95d.el9.x86 4.7 MB/s | 638 kB 00:00 2024-04-10T02:54:20.715 INFO:teuthology.orchestra.run.smithi152.stdout:(4/24): libradosstriper1-17.2.7-825.g896bb95d.e 3.4 MB/s | 457 kB 00:00 2024-04-10T02:54:21.382 INFO:teuthology.orchestra.run.smithi152.stdout:(5/24): ceph-radosgw-17.2.7-825.g896bb95d.el9.x 5.3 MB/s | 14 MB 00:02 2024-04-10T02:54:21.474 INFO:teuthology.orchestra.run.smithi152.stdout:(6/24): librgw2-17.2.7-825.g896bb95d.el9.x86_64 6.5 MB/s | 4.9 MB 00:00 2024-04-10T02:54:21.491 INFO:teuthology.orchestra.run.smithi152.stdout:(7/24): python3-ceph-argparse-17.2.7-825.g896bb 416 kB/s | 45 kB 00:00 2024-04-10T02:54:21.558 INFO:teuthology.orchestra.run.smithi152.stdout:(8/24): python3-ceph-common-17.2.7-825.g896bb95 1.4 MB/s | 118 kB 00:00 2024-04-10T02:54:21.583 INFO:teuthology.orchestra.run.smithi152.stdout:(9/24): python3-cephfs-17.2.7-825.g896bb95d.el9 1.7 MB/s | 156 kB 00:00 2024-04-10T02:54:21.611 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:54:21.650 INFO:teuthology.orchestra.run.smithi152.stdout:(10/24): python3-rados-17.2.7-825.g896bb95d.el9 3.4 MB/s | 320 kB 00:00 2024-04-10T02:54:21.675 INFO:teuthology.orchestra.run.smithi152.stdout:(11/24): python3-rbd-17.2.7-825.g896bb95d.el9.x 3.1 MB/s | 292 kB 00:00 2024-04-10T02:54:21.726 INFO:teuthology.orchestra.run.smithi152.stdout:(12/24): python3-rgw-17.2.7-825.g896bb95d.el9.x 1.3 MB/s | 100 kB 00:00 2024-04-10T02:54:21.929 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-10T02:54:21.951 INFO:teuthology.orchestra.run.smithi152.stdout:(13/24): ceph-common-17.2.7-825.g896bb95d.el9.x 6.9 MB/s | 22 MB 00:03 2024-04-10T02:54:21.975 INFO:teuthology.orchestra.run.smithi102.stdout: Upgrading : librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:22.027 INFO:teuthology.orchestra.run.smithi152.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 151 kB/s | 45 kB 00:00 2024-04-10T02:54:22.051 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:22.060 INFO:teuthology.orchestra.run.smithi152.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 417 kB/s | 160 kB 00:00 2024-04-10T02:54:22.102 INFO:teuthology.orchestra.run.smithi152.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-04-10T02:54:22.129 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 3/26 2024-04-10T02:54:22.169 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:22.177 INFO:teuthology.orchestra.run.smithi152.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.9 MB/s | 292 kB 00:00 2024-04-10T02:54:22.257 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:22.269 INFO:teuthology.orchestra.run.smithi152.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.0 MB/s | 662 kB 00:00 2024-04-10T02:54:22.309 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-10T02:54:22.336 INFO:teuthology.orchestra.run.smithi152.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.3 MB/s | 309 kB 00:00 2024-04-10T02:54:22.348 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-10T02:54:22.353 INFO:teuthology.orchestra.run.smithi152.stdout:(20/24): liboath-2.6.7-2.el9.x86_64.rpm 279 kB/s | 49 kB 00:00 2024-04-10T02:54:22.378 INFO:teuthology.orchestra.run.smithi152.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 620 kB/s | 67 kB 00:00 2024-04-10T02:54:22.453 INFO:teuthology.orchestra.run.smithi152.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 14 MB/s | 1.6 MB 00:00 2024-04-10T02:54:22.720 INFO:teuthology.orchestra.run.smithi152.stdout:(23/24): librbd1-17.2.7-825.g896bb95d.el9.x86_6 8.6 MB/s | 2.9 MB 00:00 2024-04-10T02:54:22.770 INFO:teuthology.orchestra.run.smithi152.stdout:(24/24): librados2-17.2.7-825.g896bb95d.el9.x86 7.9 MB/s | 3.1 MB 00:00 2024-04-10T02:54:22.775 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:22.775 INFO:teuthology.orchestra.run.smithi152.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-10T02:54:22.784 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-10T02:54:22.813 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 8/26 2024-04-10T02:54:22.835 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:54:22.868 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:54:22.868 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:54:22.876 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 8/26 2024-04-10T02:54:22.904 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-10T02:54:22.947 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x 10/26 2024-04-10T02:54:22.984 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 11/26 2024-04-10T02:54:23.046 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 12/26 2024-04-10T02:54:23.074 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:23.132 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:23.171 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-10T02:54:23.216 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-10T02:54:23.249 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-10T02:54:23.334 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:54:23.335 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:54:23.563 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-10T02:54:23.586 INFO:teuthology.orchestra.run.smithi102.stdout: Upgrading : librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 18/26 2024-04-10T02:54:23.676 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 18/26 2024-04-10T02:54:23.750 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 19/26 2024-04-10T02:54:23.780 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86 20/26 2024-04-10T02:54:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:54:24.991 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-10T02:54:25.031 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:25.125 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:25.210 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 3/26 2024-04-10T02:54:25.258 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:25.331 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:25.384 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-10T02:54:25.413 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-10T02:54:25.845 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-10T02:54:25.869 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 8/26 2024-04-10T02:54:25.932 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 8/26 2024-04-10T02:54:25.969 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-10T02:54:26.012 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x 10/26 2024-04-10T02:54:26.049 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 11/26 2024-04-10T02:54:26.111 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 12/26 2024-04-10T02:54:26.139 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:26.164 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:26.202 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:26.205 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:26.260 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-10T02:54:26.306 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-10T02:54:26.338 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-10T02:54:26.633 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-10T02:54:26.660 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 18/26 2024-04-10T02:54:26.732 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:26.733 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 18/26 2024-04-10T02:54:26.772 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 22/26 2024-04-10T02:54:26.806 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 19/26 2024-04-10T02:54:26.837 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86 20/26 2024-04-10T02:54:26.863 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 22/26 2024-04-10T02:54:26.863 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-10T02:54:26.863 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-10T02:54:26.863 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:26.889 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:29.270 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:29.309 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:29.841 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 21/26 2024-04-10T02:54:29.896 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 22/26 2024-04-10T02:54:29.997 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 22/26 2024-04-10T02:54:29.997 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-10T02:54:29.997 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-10T02:54:29.997 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:30.115 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:36.337 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:36.337 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /sys 2024-04-10T02:54:36.337 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /proc 2024-04-10T02:54:36.337 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /mnt 2024-04-10T02:54:36.337 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /var/tmp 2024-04-10T02:54:36.338 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /home 2024-04-10T02:54:36.338 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /root 2024-04-10T02:54:36.338 INFO:teuthology.orchestra.run.smithi102.stdout:skipping the directory /tmp 2024-04-10T02:54:36.338 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:37.629 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 24/26 2024-04-10T02:54:37.715 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 24/26 2024-04-10T02:54:37.715 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:54:37.715 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-10T02:54:37.715 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-10T02:54:37.716 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-10T02:54:37.716 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:37.717 INFO:teuthology.orchestra.run.smithi102.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-10T02:54:37.807 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-10T02:54:37.807 INFO:teuthology.orchestra.run.smithi102.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:39.063 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 1/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 3/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 5/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 6/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 7/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x 8/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86 9/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 10/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 11/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 12/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-10T02:54:39.064 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librados2-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-10T02:54:39.065 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 25/26 2024-04-10T02:54:39.553 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:39.553 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /sys 2024-04-10T02:54:39.553 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /proc 2024-04-10T02:54:39.553 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /mnt 2024-04-10T02:54:39.554 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /var/tmp 2024-04-10T02:54:39.554 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /home 2024-04-10T02:54:39.554 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /root 2024-04-10T02:54:39.554 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /tmp 2024-04-10T02:54:39.554 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout:Upgraded: 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.179 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.180 INFO:teuthology.orchestra.run.smithi102.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:40.181 INFO:teuthology.orchestra.run.smithi102.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-10T02:54:40.182 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:40.182 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:54:40.528 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-test 2024-04-10T02:54:40.882 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 24/26 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 24/26 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-10T02:54:40.960 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:40.962 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-10T02:54:41.043 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-10T02:54:41.043 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:41.123 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:29 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:54:41.412 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.413 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.414 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.415 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.416 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.417 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.418 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:41.419 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:41.420 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.421 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.422 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:41.422 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.422 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.422 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.422 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:41.487 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-test x86_64 2:17.2.7-825.g896bb95d.el9 ceph 48 M 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-10T02:54:41.488 INFO:teuthology.orchestra.run.smithi102.stdout: libcephsqlite x86_64 2:17.2.7-825.g896bb95d.el9 ceph 163 k 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout:Install 6 Packages 2024-04-10T02:54:41.489 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:41.490 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 49 M 2024-04-10T02:54:41.490 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 193 M 2024-04-10T02:54:41.490 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:54:41.845 INFO:teuthology.orchestra.run.smithi102.stdout:(1/6): jq-1.6-16.el9.x86_64.rpm 1.0 MB/s | 187 kB 00:00 2024-04-10T02:54:41.912 INFO:teuthology.orchestra.run.smithi102.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 3.2 MB/s | 218 kB 00:00 2024-04-10T02:54:42.021 INFO:teuthology.orchestra.run.smithi102.stdout:(3/6): libcephsqlite-17.2.7-825.g896bb95d.el9.x 461 kB/s | 163 kB 00:00 2024-04-10T02:54:42.163 INFO:teuthology.orchestra.run.smithi102.stdout:(4/6): socat-1.7.4.1-5.el9.x86_64.rpm 1.2 MB/s | 305 kB 00:00 2024-04-10T02:54:42.190 INFO:teuthology.orchestra.run.smithi102.stdout:(5/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 381 kB/s | 64 kB 00:00 2024-04-10T02:54:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 1/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 3/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 4/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 5/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 6/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 7/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x 8/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86 9/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 10/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 11/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 12/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 13/26 2024-04-10T02:54:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:17.2.7-825.g896bb95d.el9.x86_64 23/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-10T02:54:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 25/26 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: librados2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-10T02:54:43.071 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-10T02:54:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:43.073 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:54:43.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-test 2024-04-10T02:54:43.997 INFO:teuthology.orchestra.run.smithi102.stdout:(6/6): ceph-test-17.2.7-825.g896bb95d.el9.x86_6 21 MB/s | 48 MB 00:02 2024-04-10T02:54:43.999 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:43.999 INFO:teuthology.orchestra.run.smithi102.stdout:Total 19 MB/s | 49 MB 00:02 2024-04-10T02:54:44.035 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:29 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:54:44.047 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:54:44.065 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:54:44.065 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:54:44.326 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.327 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:54:44.328 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:54:44.329 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.330 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.331 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.332 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:44.333 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.334 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.335 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:44.394 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test x86_64 2:17.2.7-825.g896bb95d.el9 ceph 48 M 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite x86_64 2:17.2.7-825.g896bb95d.el9 ceph 163 k 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-10T02:54:44.395 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 49 M 2024-04-10T02:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 193 M 2024-04-10T02:54:44.397 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:54:44.455 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:54:44.455 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:54:44.792 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): libcephsqlite-17.2.7-825.g896bb95d.el9.x 697 kB/s | 163 kB 00:00 2024-04-10T02:54:44.826 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): jq-1.6-16.el9.x86_64.rpm 701 kB/s | 187 kB 00:00 2024-04-10T02:54:45.034 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:54:45.076 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 768 kB/s | 218 kB 00:00 2024-04-10T02:54:45.101 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): socat-1.7.4.1-5.el9.x86_64.rpm 1.1 MB/s | 305 kB 00:00 2024-04-10T02:54:45.123 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-10T02:54:45.165 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-10T02:54:45.196 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-10T02:54:45.238 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-10T02:54:45.259 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 348 kB/s | 64 kB 00:00 2024-04-10T02:54:45.275 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 5/6 2024-04-10T02:54:47.368 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): ceph-test-17.2.7-825.g896bb95d.el9.x86_6 17 MB/s | 48 MB 00:02 2024-04-10T02:54:47.370 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:47.370 INFO:teuthology.orchestra.run.smithi152.stdout:Total 16 MB/s | 49 MB 00:02 2024-04-10T02:54:47.420 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:54:47.440 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:54:47.440 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:54:47.832 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:54:47.833 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:54:48.409 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:54:48.478 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-10T02:54:48.514 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-10T02:54:48.542 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-10T02:54:48.584 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-10T02:54:48.613 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 5/6 2024-04-10T02:54:49.661 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 5/6 2024-04-10T02:54:49.740 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 6/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 6/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 1/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 2/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-10T02:54:52.031 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: jq-1.6-16.el9.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:52.430 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:54:52.730 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph 2024-04-10T02:54:53.043 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 5/6 2024-04-10T02:54:53.128 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 6/6 2024-04-10T02:54:53.325 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:41 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:54:53.612 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:54:53.613 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:54:53.614 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:54:53.615 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.616 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.617 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.618 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.619 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.620 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:53.621 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.621 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.621 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.621 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:53.685 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:54:53.687 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph x86_64 2:17.2.7-825.g896bb95d.el9 ceph 6.5 k 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mds x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.1 M 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr x86_64 2:17.2.7-825.g896bb95d.el9 ceph 1.4 M 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-modules-core noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 240 k 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mon x86_64 2:17.2.7-825.g896bb95d.el9 ceph 3.9 M 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-osd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 15 M 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-10T02:54:53.688 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-10T02:54:53.689 INFO:teuthology.orchestra.run.smithi102.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:54:53.690 INFO:teuthology.orchestra.run.smithi102.stdout:Install 33 Packages 2024-04-10T02:54:53.691 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:54:53.691 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 27 M 2024-04-10T02:54:53.691 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 96 M 2024-04-10T02:54:53.691 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:54:54.526 INFO:teuthology.orchestra.run.smithi102.stdout:(1/33): ceph-17.2.7-825.g896bb95d.el9.x86_64.rp 27 kB/s | 6.5 kB 00:00 2024-04-10T02:54:54.643 INFO:teuthology.orchestra.run.smithi102.stdout:(2/33): ceph-mds-17.2.7-825.g896bb95d.el9.x86_6 5.7 MB/s | 2.1 MB 00:00 2024-04-10T02:54:54.843 INFO:teuthology.orchestra.run.smithi102.stdout:(3/33): ceph-mgr-17.2.7-825.g896bb95d.el9.x86_6 2.5 MB/s | 1.4 MB 00:00 2024-04-10T02:54:54.943 INFO:teuthology.orchestra.run.smithi102.stdout:(4/33): ceph-mgr-modules-core-17.2.7-825.g896bb 2.3 MB/s | 240 kB 00:00 2024-04-10T02:54:55.118 INFO:teuthology.orchestra.run.smithi102.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-04-10T02:54:55.411 INFO:teuthology.orchestra.run.smithi102.stdout:(6/33): ceph-osd-17.2.7-825.g896bb95d.el9.x86_6 20 MB/s | 15 MB 00:00 2024-04-10T02:54:55.413 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 6/6 2024-04-10T02:54:55.413 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 1/6 2024-04-10T02:54:55.414 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 2/6 2024-04-10T02:54:55.414 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-10T02:54:55.414 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-10T02:54:55.414 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-10T02:54:55.536 INFO:teuthology.orchestra.run.smithi102.stdout:(7/33): python3-cryptography-36.0.1-4.el9.x86_6 2.9 MB/s | 1.2 MB 00:00 2024-04-10T02:54:55.578 INFO:teuthology.orchestra.run.smithi102.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 639 kB/s | 106 kB 00:00 2024-04-10T02:54:55.670 INFO:teuthology.orchestra.run.smithi102.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 1.3 MB/s | 125 kB 00:00 2024-04-10T02:54:55.728 INFO:teuthology.orchestra.run.smithi102.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 703 kB/s | 135 kB 00:00 2024-04-10T02:54:55.754 INFO:teuthology.orchestra.run.smithi102.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.5 MB/s | 215 kB 00:00 2024-04-10T02:54:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-10T02:54:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:55.856 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: jq-1.6-16.el9.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:55.857 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:54:55.920 INFO:teuthology.orchestra.run.smithi102.stdout:(12/33): ceph-mon-17.2.7-825.g896bb95d.el9.x86_ 2.8 MB/s | 3.9 MB 00:01 2024-04-10T02:54:55.979 INFO:teuthology.orchestra.run.smithi102.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 154 kB/s | 35 kB 00:00 2024-04-10T02:54:56.029 INFO:teuthology.orchestra.run.smithi102.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 572 kB/s | 172 kB 00:00 2024-04-10T02:54:56.071 INFO:teuthology.orchestra.run.smithi102.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 611 kB/s | 42 kB 00:00 2024-04-10T02:54:56.122 INFO:teuthology.orchestra.run.smithi102.stdout:(16/33): python3-pytz-2021.1-5.el9.noarch.rpm 255 kB/s | 51 kB 00:00 2024-04-10T02:54:56.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph 2024-04-10T02:54:56.289 INFO:teuthology.orchestra.run.smithi102.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 168 kB/s | 43 kB 00:00 2024-04-10T02:54:56.364 INFO:teuthology.orchestra.run.smithi102.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 143 kB/s | 11 kB 00:00 2024-04-10T02:54:56.447 INFO:teuthology.orchestra.run.smithi102.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 212 kB/s | 18 kB 00:00 2024-04-10T02:54:56.472 INFO:teuthology.orchestra.run.smithi102.stdout:(20/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 428 kB/s | 172 kB 00:00 2024-04-10T02:54:56.531 INFO:teuthology.orchestra.run.smithi102.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 277 kB/s | 23 kB 00:00 2024-04-10T02:54:56.556 INFO:teuthology.orchestra.run.smithi102.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 232 kB/s | 19 kB 00:00 2024-04-10T02:54:56.581 INFO:teuthology.orchestra.run.smithi102.stdout:(23/33): python3-cherrypy-18.6.1-2.el9.noarch.r 821 kB/s | 358 kB 00:00 2024-04-10T02:54:56.615 INFO:teuthology.orchestra.run.smithi102.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 234 kB/s | 20 kB 00:00 2024-04-10T02:54:56.641 INFO:teuthology.orchestra.run.smithi102.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 551 kB/s | 46 kB 00:00 2024-04-10T02:54:56.666 INFO:teuthology.orchestra.run.smithi102.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 937 kB/s | 79 kB 00:00 2024-04-10T02:54:56.716 INFO:teuthology.orchestra.run.smithi102.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 219 kB/s | 16 kB 00:00 2024-04-10T02:54:56.741 INFO:teuthology.orchestra.run.smithi102.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-04-10T02:54:56.755 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:41 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:54:56.792 INFO:teuthology.orchestra.run.smithi102.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 474 kB/s | 36 kB 00:00 2024-04-10T02:54:56.825 INFO:teuthology.orchestra.run.smithi102.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.3 MB/s | 272 kB 00:00 2024-04-10T02:54:56.851 INFO:teuthology.orchestra.run.smithi102.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-04-10T02:54:56.893 INFO:teuthology.orchestra.run.smithi102.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 296 kB/s | 20 kB 00:00 2024-04-10T02:54:56.959 INFO:teuthology.orchestra.run.smithi102.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-10T02:54:56.965 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:56.965 INFO:teuthology.orchestra.run.smithi102.stdout:Total 8.3 MB/s | 27 MB 00:03 2024-04-10T02:54:57.049 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:54:57.049 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:54:57.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:54:57.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:54:57.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:54:57.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.056 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.057 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:54:57.076 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:54:57.113 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:54:57.113 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:54:57.131 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:54:57.133 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:57.133 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:54:57.133 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:57.133 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:54:57.133 INFO:teuthology.orchestra.run.smithi152.stdout: ceph x86_64 2:17.2.7-825.g896bb95d.el9 ceph 6.5 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.1 M 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr x86_64 2:17.2.7-825.g896bb95d.el9 ceph 1.4 M 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 240 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon x86_64 2:17.2.7-825.g896bb95d.el9 ceph 3.9 M 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 15 M 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-10T02:54:57.134 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-10T02:54:57.135 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout:Install 33 Packages 2024-04-10T02:54:57.136 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:54:57.137 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 27 M 2024-04-10T02:54:57.137 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 96 M 2024-04-10T02:54:57.137 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:54:57.457 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:54:57.457 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:54:57.911 INFO:teuthology.orchestra.run.smithi152.stdout:(1/33): ceph-17.2.7-825.g896bb95d.el9.x86_64.rp 38 kB/s | 6.5 kB 00:00 2024-04-10T02:54:58.169 INFO:teuthology.orchestra.run.smithi152.stdout:(2/33): ceph-mgr-17.2.7-825.g896bb95d.el9.x86_6 3.3 MB/s | 1.4 MB 00:00 2024-04-10T02:54:58.220 INFO:teuthology.orchestra.run.smithi152.stdout:(3/33): ceph-mds-17.2.7-825.g896bb95d.el9.x86_6 4.3 MB/s | 2.1 MB 00:00 2024-04-10T02:54:58.286 INFO:teuthology.orchestra.run.smithi152.stdout:(4/33): ceph-mgr-modules-core-17.2.7-825.g896bb 3.5 MB/s | 240 kB 00:00 2024-04-10T02:54:58.462 INFO:teuthology.orchestra.run.smithi152.stdout:(5/33): ceph-mon-17.2.7-825.g896bb95d.el9.x86_6 7.1 MB/s | 3.9 MB 00:00 2024-04-10T02:54:58.687 INFO:teuthology.orchestra.run.smithi152.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 633 kB/s | 253 kB 00:00 2024-04-10T02:54:58.751 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:54:58.804 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-10T02:54:58.836 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-10T02:54:58.845 INFO:teuthology.orchestra.run.smithi152.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 673 kB/s | 106 kB 00:00 2024-04-10T02:54:58.869 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-10T02:54:58.962 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-10T02:54:58.987 INFO:teuthology.orchestra.run.smithi152.stdout:(8/33): python3-cryptography-36.0.1-4.el9.x86_6 2.3 MB/s | 1.2 MB 00:00 2024-04-10T02:54:59.004 INFO:teuthology.orchestra.run.smithi152.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 853 kB/s | 135 kB 00:00 2024-04-10T02:54:59.039 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-10T02:54:59.080 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-10T02:54:59.123 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-10T02:54:59.153 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-10T02:54:59.163 INFO:teuthology.orchestra.run.smithi152.stdout:(10/33): ceph-osd-17.2.7-825.g896bb95d.el9.x86_ 15 MB/s | 15 MB 00:00 2024-04-10T02:54:59.187 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-10T02:54:59.221 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-10T02:54:59.230 INFO:teuthology.orchestra.run.smithi152.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 952 kB/s | 215 kB 00:00 2024-04-10T02:54:59.247 INFO:teuthology.orchestra.run.smithi152.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 482 kB/s | 125 kB 00:00 2024-04-10T02:54:59.262 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-10T02:54:59.289 INFO:teuthology.orchestra.run.smithi152.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-04-10T02:54:59.296 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-10T02:54:59.314 INFO:teuthology.orchestra.run.smithi152.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 415 kB/s | 35 kB 00:00 2024-04-10T02:54:59.331 INFO:teuthology.orchestra.run.smithi152.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 993 kB/s | 42 kB 00:00 2024-04-10T02:54:59.331 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-10T02:54:59.348 INFO:teuthology.orchestra.run.smithi152.stdout:(16/33): python3-pytz-2021.1-5.el9.noarch.rpm 505 kB/s | 51 kB 00:00 2024-04-10T02:54:59.369 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-10T02:54:59.452 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-10T02:54:59.481 INFO:teuthology.orchestra.run.smithi152.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 260 kB/s | 43 kB 00:00 2024-04-10T02:54:59.540 INFO:teuthology.orchestra.run.smithi152.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 185 kB/s | 11 kB 00:00 2024-04-10T02:54:59.565 INFO:teuthology.orchestra.run.smithi152.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 734 kB/s | 172 kB 00:00 2024-04-10T02:54:59.588 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-10T02:54:59.599 INFO:teuthology.orchestra.run.smithi152.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 301 kB/s | 18 kB 00:00 2024-04-10T02:54:59.624 INFO:teuthology.orchestra.run.smithi152.stdout:(21/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-04-10T02:54:59.649 INFO:teuthology.orchestra.run.smithi152.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 277 kB/s | 23 kB 00:00 2024-04-10T02:54:59.669 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-10T02:54:59.674 INFO:teuthology.orchestra.run.smithi152.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 258 kB/s | 19 kB 00:00 2024-04-10T02:54:59.700 INFO:teuthology.orchestra.run.smithi152.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 260 kB/s | 20 kB 00:00 2024-04-10T02:54:59.725 INFO:teuthology.orchestra.run.smithi152.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 612 kB/s | 46 kB 00:00 2024-04-10T02:54:59.730 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-10T02:54:59.759 INFO:teuthology.orchestra.run.smithi152.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 938 kB/s | 79 kB 00:00 2024-04-10T02:54:59.768 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-10T02:54:59.784 INFO:teuthology.orchestra.run.smithi152.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 281 kB/s | 16 kB 00:00 2024-04-10T02:54:59.809 INFO:teuthology.orchestra.run.smithi152.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.4 MB/s | 272 kB 00:00 2024-04-10T02:54:59.817 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-10T02:54:59.835 INFO:teuthology.orchestra.run.smithi152.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-04-10T02:54:59.860 INFO:teuthology.orchestra.run.smithi152.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 473 kB/s | 36 kB 00:00 2024-04-10T02:54:59.872 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-10T02:54:59.885 INFO:teuthology.orchestra.run.smithi152.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-04-10T02:54:59.919 INFO:teuthology.orchestra.run.smithi152.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 342 kB/s | 20 kB 00:00 2024-04-10T02:54:59.969 INFO:teuthology.orchestra.run.smithi152.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.1 MB/s | 427 kB 00:00 2024-04-10T02:54:59.971 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:54:59.971 INFO:teuthology.orchestra.run.smithi152.stdout:Total 9.6 MB/s | 27 MB 00:02 2024-04-10T02:55:00.068 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-10T02:55:00.093 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:00.123 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-10T02:55:00.129 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:00.129 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:00.200 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-10T02:55:00.290 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-10T02:55:00.330 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-10T02:55:00.446 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-10T02:55:00.479 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:00.480 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:00.592 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.n 28/33 2024-04-10T02:55:00.624 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 29/33 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 29/33 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-10T02:55:00.649 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:01.747 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:01.813 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-10T02:55:01.845 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-10T02:55:01.878 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-10T02:55:01.971 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-10T02:55:02.017 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 30/33 2024-04-10T02:55:02.041 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 30/33 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-10T02:55:02.046 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:02.090 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-10T02:55:02.133 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-10T02:55:02.162 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-10T02:55:02.197 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-10T02:55:02.230 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-10T02:55:02.279 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-10T02:55:02.314 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-10T02:55:02.372 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-10T02:55:02.411 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-10T02:55:02.418 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 31/33 2024-04-10T02:55:02.441 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 31/33 2024-04-10T02:55:02.441 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:02.441 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-10T02:55:02.442 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-10T02:55:02.442 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-10T02:55:02.442 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:02.485 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-10T02:55:02.639 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-10T02:55:02.644 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 32/33 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 32/33 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-10T02:55:02.667 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:02.719 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-10T02:55:02.733 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-2:17.2.7-825.g896bb95d.el9.x86_64 33/33 2024-04-10T02:55:02.772 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-10T02:55:02.811 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-10T02:55:02.860 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-10T02:55:02.915 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-10T02:55:03.113 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-10T02:55:03.165 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-10T02:55:03.244 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-10T02:55:03.331 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-10T02:55:03.371 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-10T02:55:03.485 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-10T02:55:03.634 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.n 28/33 2024-04-10T02:55:03.665 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 29/33 2024-04-10T02:55:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 29/33 2024-04-10T02:55:03.692 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:03.692 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-10T02:55:03.692 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-10T02:55:03.692 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-10T02:55:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:04.317 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-2:17.2.7-825.g896bb95d.el9.x86_64 33/33 2024-04-10T02:55:04.317 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-2:17.2.7-825.g896bb95d.el9.x86_64 1/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 2/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 3/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 4/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 5/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.n 6/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-10T02:55:04.318 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-10T02:55:04.321 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-10T02:55:04.322 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-10T02:55:04.323 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-10T02:55:04.323 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-10T02:55:05.008 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 30/33 2024-04-10T02:55:05.034 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 30/33 2024-04-10T02:55:05.035 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:05.035 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-10T02:55:05.035 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-10T02:55:05.035 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-10T02:55:05.035 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:05.276 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-10T02:55:05.277 INFO:teuthology.orchestra.run.smithi102.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-ply-3.11-14.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-10T02:55:05.278 INFO:teuthology.orchestra.run.smithi102.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-10T02:55:05.279 INFO:teuthology.orchestra.run.smithi102.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-10T02:55:05.279 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:05.279 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:05.400 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 31/33 2024-04-10T02:55:05.424 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 31/33 2024-04-10T02:55:05.424 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:05.424 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-10T02:55:05.425 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-10T02:55:05.425 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-10T02:55:05.425 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:05.626 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 32/33 2024-04-10T02:55:05.638 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-base 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 32/33 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-10T02:55:05.649 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:05.714 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-2:17.2.7-825.g896bb95d.el9.x86_64 33/33 2024-04-10T02:55:06.231 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:54 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.521 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.522 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:06.523 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:06.524 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:06.530 INFO:teuthology.orchestra.run.smithi102.stdout:Package ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:06.586 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:06.588 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:06.588 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:06.651 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install cephadm 2024-04-10T02:55:07.238 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:55 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-2:17.2.7-825.g896bb95d.el9.x86_64 33/33 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-2:17.2.7-825.g896bb95d.el9.x86_64 1/33 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 2/33 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 3/33 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 4/33 2024-04-10T02:55:07.467 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 5/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.n 6/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-10T02:55:07.468 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-10T02:55:07.469 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-10T02:55:07.470 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-10T02:55:07.470 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-10T02:55:07.470 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-10T02:55:07.470 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-10T02:55:07.470 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-10T02:55:07.524 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:07.525 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:07.526 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:07.527 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:07.528 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.529 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.530 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:07.531 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.532 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.533 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:07.591 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout: cephadm noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 82 k 2024-04-10T02:55:07.592 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 82 k 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 372 k 2024-04-10T02:55:07.593 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:07.918 INFO:teuthology.orchestra.run.smithi102.stdout:cephadm-17.2.7-825.g896bb95d.el9.noarch.rpm 253 kB/s | 82 kB 00:00 2024-04-10T02:55:07.918 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:07.919 INFO:teuthology.orchestra.run.smithi102.stdout:Total 252 kB/s | 82 kB 00:00 2024-04-10T02:55:07.919 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:07.923 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:07.924 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:07.925 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:07.926 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:08.007 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-10T02:55:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply-3.11-14.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-10T02:55:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:08.378 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:08.425 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:08.522 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:08.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-base 2024-04-10T02:55:08.884 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:09.178 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:09.179 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:09.179 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:09.179 INFO:teuthology.orchestra.run.smithi102.stdout: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:09.179 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:09.179 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:09.241 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:54 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:09.378 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-immutable-object-cache 2024-04-10T02:55:09.527 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:09.527 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:09.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:09.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:09.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:09.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:09.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:09.536 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-base-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:09.593 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:09.594 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:55:09.595 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:09.666 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install cephadm 2024-04-10T02:55:09.984 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:57 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:10.254 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:55 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:10.280 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:10.281 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.282 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:10.283 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:10.284 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.285 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.286 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.287 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.288 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.348 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:10.348 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repo Size 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-825.g896bb95d.el9 ceph 137 k 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:55:10.349 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:10.350 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 137 k 2024-04-10T02:55:10.350 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 379 k 2024-04-10T02:55:10.350 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:10.544 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:10.544 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:10.544 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:10.544 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.545 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:10.546 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.547 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:10.548 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:10.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:10.610 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 82 k 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:10.611 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:10.612 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:55:10.612 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:10.612 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-10T02:55:10.612 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 372 k 2024-04-10T02:55:10.612 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:10.867 INFO:teuthology.orchestra.run.smithi102.stdout:ceph-immutable-object-cache-17.2.7-825.g896bb95 266 kB/s | 137 kB 00:00 2024-04-10T02:55:10.867 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:10.867 INFO:teuthology.orchestra.run.smithi102.stdout:Total 265 kB/s | 137 kB 00:00 2024-04-10T02:55:10.868 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:10.876 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:10.876 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:10.929 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:10.930 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:11.066 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:11.117 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:11.142 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:11.142 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:11.142 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-10T02:55:11.142 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:11.348 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm-17.2.7-825.g896bb95d.el9.noarch.rpm 111 kB/s | 82 kB 00:00 2024-04-10T02:55:11.349 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:11.349 INFO:teuthology.orchestra.run.smithi152.stdout:Total 111 kB/s | 82 kB 00:00 2024-04-10T02:55:11.349 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:11.354 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:11.355 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:11.356 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:11.357 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:11.432 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:11.814 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:11.903 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:12.195 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:12.196 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:12.196 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:12.196 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:12.196 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:12.196 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:12.464 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-mgr 2024-04-10T02:55:12.590 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/1 2024-04-10T02:55:12.590 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:12.590 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:12.590 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:12.591 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:12.591 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:12.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-immutable-object-cache 2024-04-10T02:55:13.061 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:01 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:13.347 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:13.347 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:13.347 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:13.347 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.348 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:13.349 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.350 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:13.351 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.352 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.353 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.354 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.355 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.356 INFO:teuthology.orchestra.run.smithi102.stdout:Package ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:13.413 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:13.414 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:13.414 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:13.426 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:58 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:13.483 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-mgr-dashboard 2024-04-10T02:55:13.714 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:13.714 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:13.714 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:13.714 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:13.715 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:13.716 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.717 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.718 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:13.719 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.720 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.721 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:13.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:13.782 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-825.g896bb95d.el9 ceph 137 k 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:13.783 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 137 k 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 379 k 2024-04-10T02:55:13.784 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:14.074 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:02 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:14.084 INFO:teuthology.orchestra.run.smithi152.stdout:ceph-immutable-object-cache-17.2.7-825.g896bb95 457 kB/s | 137 kB 00:00 2024-04-10T02:55:14.084 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:14.085 INFO:teuthology.orchestra.run.smithi152.stdout:Total 455 kB/s | 137 kB 00:00 2024-04-10T02:55:14.085 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:14.092 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:14.092 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:14.146 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:14.146 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:14.291 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:14.343 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.365 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:14.366 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.367 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:14.368 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.369 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:14.370 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.371 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.372 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.373 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:55:14.374 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-10T02:55:14.375 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:14.434 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-dashboard noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 1.7 M 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-grafana-dashboards noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 24 k 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-prometheus-alerts noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 15 k 2024-04-10T02:55:14.435 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout:Installing weak dependencies: 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout:Install 7 Packages 2024-04-10T02:55:14.436 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:14.437 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 2.0 M 2024-04-10T02:55:14.437 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 8.1 M 2024-04-10T02:55:14.437 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:14.713 INFO:teuthology.orchestra.run.smithi102.stdout:(1/7): ceph-prometheus-alerts-17.2.7-825.g896bb 74 kB/s | 15 kB 00:00 2024-04-10T02:55:14.805 INFO:teuthology.orchestra.run.smithi102.stdout:(2/7): ceph-grafana-dashboards-17.2.7-825.g896b 84 kB/s | 24 kB 00:00 2024-04-10T02:55:15.030 INFO:teuthology.orchestra.run.smithi102.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 29 kB/s | 9.0 kB 00:00 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.e 1/1 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:15.306 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:15.330 INFO:teuthology.orchestra.run.smithi102.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 102 kB/s | 31 kB 00:00 2024-04-10T02:55:15.380 INFO:teuthology.orchestra.run.smithi102.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 71 kB/s | 41 kB 00:00 2024-04-10T02:55:15.564 INFO:teuthology.orchestra.run.smithi102.stdout:(6/7): ceph-mgr-dashboard-17.2.7-825.g896bb95d. 1.6 MB/s | 1.7 MB 00:01 2024-04-10T02:55:15.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr 2024-04-10T02:55:15.806 INFO:teuthology.orchestra.run.smithi102.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 397 kB/s | 188 kB 00:00 2024-04-10T02:55:15.808 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:15.808 INFO:teuthology.orchestra.run.smithi102.stdout:Total 1.5 MB/s | 2.0 MB 00:01 2024-04-10T02:55:15.808 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:15.817 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:15.817 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:15.887 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:15.887 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:16.177 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:16.203 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:01 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:16.232 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-10T02:55:16.266 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-10T02:55:16.329 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-10T02:55:16.361 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-10T02:55:16.390 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.no 5/7 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.494 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:16.495 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.496 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.497 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.498 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:16.499 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.500 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.501 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.502 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:16.503 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-mgr-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:16.560 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:16.561 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:55:16.561 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:16.616 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.n 6/7 2024-04-10T02:55:16.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-dashboard 2024-04-10T02:55:16.644 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 7/7 2024-04-10T02:55:17.171 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 7/7 2024-04-10T02:55:17.171 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.n 1/7 2024-04-10T02:55:17.172 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 2/7 2024-04-10T02:55:17.172 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.no 3/7 2024-04-10T02:55:17.172 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-10T02:55:17.172 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-10T02:55:17.172 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-10T02:55:17.238 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:02 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:17.484 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-10T02:55:17.485 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-10T02:55:17.485 INFO:teuthology.orchestra.run.smithi102.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-10T02:55:17.485 INFO:teuthology.orchestra.run.smithi102.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-10T02:55:17.485 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:17.485 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:17.528 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:17.528 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:17.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:17.528 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.528 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:17.529 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:17.530 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:17.531 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:17.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:17.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:17.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:17.597 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 1.7 M 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 24 k 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 15 k 2024-04-10T02:55:17.598 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout:Installing weak dependencies: 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout:Install 7 Packages 2024-04-10T02:55:17.599 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:17.600 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.0 M 2024-04-10T02:55:17.600 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 8.1 M 2024-04-10T02:55:17.600 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:17.714 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-10T02:55:17.879 INFO:teuthology.orchestra.run.smithi152.stdout:(1/7): ceph-grafana-dashboards-17.2.7-825.g896b 125 kB/s | 24 kB 00:00 2024-04-10T02:55:17.904 INFO:teuthology.orchestra.run.smithi152.stdout:(2/7): ceph-prometheus-alerts-17.2.7-825.g896bb 66 kB/s | 15 kB 00:00 2024-04-10T02:55:18.096 INFO:teuthology.orchestra.run.smithi152.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 42 kB/s | 9.0 kB 00:00 2024-04-10T02:55:18.196 INFO:teuthology.orchestra.run.smithi152.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 140 kB/s | 41 kB 00:00 2024-04-10T02:55:18.221 INFO:teuthology.orchestra.run.smithi152.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 246 kB/s | 31 kB 00:00 2024-04-10T02:55:18.303 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:06 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:18.380 INFO:teuthology.orchestra.run.smithi152.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.0 MB/s | 188 kB 00:00 2024-04-10T02:55:18.592 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:18.592 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:18.592 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:18.592 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.592 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.593 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.594 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:18.595 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.596 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.597 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.598 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.599 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.600 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.601 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.601 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:18.661 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:18.662 INFO:teuthology.orchestra.run.smithi102.stdout:====================================================================================== 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout:====================================================================================== 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 7.4 M 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-10T02:55:18.663 INFO:teuthology.orchestra.run.smithi102.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout:====================================================================================== 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout:Install 12 Packages 2024-04-10T02:55:18.664 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:18.665 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 38 M 2024-04-10T02:55:18.665 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 209 M 2024-04-10T02:55:18.665 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:18.722 INFO:teuthology.orchestra.run.smithi152.stdout:(7/7): ceph-mgr-dashboard-17.2.7-825.g896bb95d. 1.6 MB/s | 1.7 MB 00:01 2024-04-10T02:55:18.725 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:18.726 INFO:teuthology.orchestra.run.smithi152.stdout:Total 1.8 MB/s | 2.0 MB 00:01 2024-04-10T02:55:18.726 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:18.734 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:18.734 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:18.801 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:18.802 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:19.111 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:19.146 INFO:teuthology.orchestra.run.smithi102.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.3 MB/s | 195 kB 00:00 2024-04-10T02:55:19.173 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-10T02:55:19.188 INFO:teuthology.orchestra.run.smithi102.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 4.0 MB/s | 803 kB 00:00 2024-04-10T02:55:19.207 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-10T02:55:19.253 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-10T02:55:19.254 INFO:teuthology.orchestra.run.smithi102.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 301 kB/s | 33 kB 00:00 2024-04-10T02:55:19.285 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-10T02:55:19.296 INFO:teuthology.orchestra.run.smithi102.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 435 kB/s | 18 kB 00:00 2024-04-10T02:55:19.314 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.no 5/7 2024-04-10T02:55:19.348 INFO:teuthology.orchestra.run.smithi102.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 704 kB/s | 35 kB 00:00 2024-04-10T02:55:19.415 INFO:teuthology.orchestra.run.smithi102.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 13 MB/s | 3.0 MB 00:00 2024-04-10T02:55:19.456 INFO:teuthology.orchestra.run.smithi102.stdout:(7/12): python3-devel-3.9.17-2.el9.x86_64.rpm 5.8 MB/s | 245 kB 00:00 2024-04-10T02:55:19.544 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.n 6/7 2024-04-10T02:55:19.565 INFO:teuthology.orchestra.run.smithi102.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 47 MB/s | 5.1 MB 00:00 2024-04-10T02:55:19.584 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 7/7 2024-04-10T02:55:19.632 INFO:teuthology.orchestra.run.smithi102.stdout:(9/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 17 MB/s | 4.7 MB 00:00 2024-04-10T02:55:19.666 INFO:teuthology.orchestra.run.smithi102.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.7 MB/s | 250 kB 00:00 2024-04-10T02:55:19.892 INFO:teuthology.orchestra.run.smithi102.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 63 MB/s | 16 MB 00:00 2024-04-10T02:55:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 7/7 2024-04-10T02:55:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.n 1/7 2024-04-10T02:55:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 2/7 2024-04-10T02:55:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.no 3/7 2024-04-10T02:55:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-10T02:55:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-10T02:55:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-10T02:55:20.429 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:20.430 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:20.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-10T02:55:21.266 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:06 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.562 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.563 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:21.564 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.565 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:21.566 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.567 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.568 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:21.569 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.570 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:21.571 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.571 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.571 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.571 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:21.632 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 7.4 M 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:55:21.633 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout:====================================================================================== 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout:Install 12 Packages 2024-04-10T02:55:21.634 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:21.635 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 38 M 2024-04-10T02:55:21.635 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 209 M 2024-04-10T02:55:21.635 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:22.012 INFO:teuthology.orchestra.run.smithi152.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 986 kB/s | 195 kB 00:00 2024-04-10T02:55:22.112 INFO:teuthology.orchestra.run.smithi152.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.6 MB/s | 803 kB 00:00 2024-04-10T02:55:22.220 INFO:teuthology.orchestra.run.smithi152.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 157 kB/s | 33 kB 00:00 2024-04-10T02:55:22.279 INFO:teuthology.orchestra.run.smithi152.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 316 kB/s | 18 kB 00:00 2024-04-10T02:55:22.298 INFO:teuthology.orchestra.run.smithi102.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-8 2.2 MB/s | 7.4 MB 00:03 2024-04-10T02:55:22.299 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:22.300 INFO:teuthology.orchestra.run.smithi102.stdout:Total 10 MB/s | 38 MB 00:03 2024-04-10T02:55:22.346 INFO:teuthology.orchestra.run.smithi152.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 528 kB/s | 35 kB 00:00 2024-04-10T02:55:22.455 INFO:teuthology.orchestra.run.smithi152.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 9.4 MB/s | 3.0 MB 00:00 2024-04-10T02:55:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:(7/12): python3-devel-3.9.17-2.el9.x86_64.rpm 3.6 MB/s | 245 kB 00:00 2024-04-10T02:55:22.657 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:22.684 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:22.684 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:22.697 INFO:teuthology.orchestra.run.smithi152.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 29 MB/s | 5.1 MB 00:00 2024-04-10T02:55:22.772 INFO:teuthology.orchestra.run.smithi152.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 3.3 MB/s | 250 kB 00:00 2024-04-10T02:55:23.164 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:23.164 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:23.364 INFO:teuthology.orchestra.run.smithi152.stdout:(10/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 27 MB/s | 16 MB 00:00 2024-04-10T02:55:23.489 INFO:teuthology.orchestra.run.smithi152.stdout:(11/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 4.1 MB/s | 4.7 MB 00:01 2024-04-10T02:55:23.916 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:23.982 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-10T02:55:24.022 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-10T02:55:24.120 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-10T02:55:24.156 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-10T02:55:24.339 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-10T02:55:24.372 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-10T02:55:24.455 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-10T02:55:25.115 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-10T02:55:25.242 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-10T02:55:26.488 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-10T02:55:27.528 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-10T02:55:27.555 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 12/12 2024-04-10T02:55:29.747 INFO:teuthology.orchestra.run.smithi152.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-8 956 kB/s | 7.4 MB 00:07 2024-04-10T02:55:29.750 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:29.750 INFO:teuthology.orchestra.run.smithi152.stdout:Total 4.7 MB/s | 38 MB 00:08 2024-04-10T02:55:30.112 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:30.140 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:30.140 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 12/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 1/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-10T02:55:30.204 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-10T02:55:30.205 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-10T02:55:30.205 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-10T02:55:30.628 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:30.628 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-10T02:55:30.669 INFO:teuthology.orchestra.run.smithi102.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:30.670 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:31.022 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-mgr-rook 2024-04-10T02:55:31.386 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:31.449 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-10T02:55:31.487 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-10T02:55:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-10T02:55:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-10T02:55:31.645 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:19 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:31.822 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-10T02:55:31.853 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-10T02:55:31.936 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:31.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:31.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:31.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.952 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.956 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.958 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:31.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:31.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:32.022 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:32.023 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:32.023 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:32.023 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-rook noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 47 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-10T02:55:32.024 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout:Install 13 Packages 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 2.2 M 2024-04-10T02:55:32.025 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 26 M 2024-04-10T02:55:32.026 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:32.606 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-10T02:55:32.732 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-10T02:55:32.771 INFO:teuthology.orchestra.run.smithi102.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 252 kB/s | 26 kB 00:00 2024-04-10T02:55:32.796 INFO:teuthology.orchestra.run.smithi102.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 151 kB/s | 19 kB 00:00 2024-04-10T02:55:32.855 INFO:teuthology.orchestra.run.smithi102.stdout:(3/13): ceph-mgr-rook-17.2.7-825.g896bb95d.el9. 256 kB/s | 47 kB 00:00 2024-04-10T02:55:32.914 INFO:teuthology.orchestra.run.smithi102.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-04-10T02:55:32.939 INFO:teuthology.orchestra.run.smithi102.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-04-10T02:55:32.964 INFO:teuthology.orchestra.run.smithi102.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-04-10T02:55:33.023 INFO:teuthology.orchestra.run.smithi102.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.6 MB/s | 279 kB 00:00 2024-04-10T02:55:33.181 INFO:teuthology.orchestra.run.smithi102.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 133 kB/s | 32 kB 00:00 2024-04-10T02:55:33.232 INFO:teuthology.orchestra.run.smithi102.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 56 kB/s | 15 kB 00:00 2024-04-10T02:55:33.390 INFO:teuthology.orchestra.run.smithi102.stdout:(10/13): python3-kubernetes-26.1.0-2.el9.noarch 4.9 MB/s | 1.0 MB 00:00 2024-04-10T02:55:33.416 INFO:teuthology.orchestra.run.smithi102.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 587 kB/s | 203 kB 00:00 2024-04-10T02:55:33.441 INFO:teuthology.orchestra.run.smithi102.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 283 kB/s | 59 kB 00:00 2024-04-10T02:55:33.466 INFO:teuthology.orchestra.run.smithi102.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 1.2 MB/s | 90 kB 00:00 2024-04-10T02:55:33.469 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:33.469 INFO:teuthology.orchestra.run.smithi102.stdout:Total 1.5 MB/s | 2.2 MB 00:01 2024-04-10T02:55:33.530 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:33.545 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:33.545 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:33.639 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:33.640 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:33.990 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-10T02:55:34.160 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:34.325 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-10T02:55:34.383 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-10T02:55:34.440 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-10T02:55:34.477 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-10T02:55:34.518 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-10T02:55:34.613 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-10T02:55:34.721 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-10T02:55:34.769 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-10T02:55:35.036 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-10T02:55:35.068 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 12/12 2024-04-10T02:55:35.479 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-10T02:55:35.533 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-10T02:55:35.567 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-10T02:55:35.612 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-10T02:55:35.637 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 13/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 13/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 1/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-10T02:55:36.441 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-10T02:55:36.442 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-10T02:55:36.442 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-10T02:55:36.442 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-10T02:55:36.442 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-10T02:55:36.815 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:36.816 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:37.044 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-mgr-cephadm 2024-04-10T02:55:37.632 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:25 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 12/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb9 1/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-10T02:55:37.726 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-10T02:55:37.727 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-10T02:55:37.727 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-10T02:55:37.727 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-10T02:55:37.727 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-10T02:55:37.727 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:37.918 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.919 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:37.920 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.921 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.922 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:37.923 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:37.924 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.925 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:37.926 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:37.987 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-cephadm noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 123 k 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:37.988 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout:Install 6 Packages 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 7.0 M 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 33 M 2024-04-10T02:55:37.989 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-10T02:55:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:38.175 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:38.409 INFO:teuthology.orchestra.run.smithi102.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.2 MB/s | 248 kB 00:00 2024-04-10T02:55:38.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-rook 2024-04-10T02:55:38.584 INFO:teuthology.orchestra.run.smithi102.stdout:(2/6): ceph-mgr-cephadm-17.2.7-825.g896bb95d.el 328 kB/s | 123 kB 00:00 2024-04-10T02:55:39.118 INFO:teuthology.orchestra.run.smithi102.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 108 kB/s | 58 kB 00:00 2024-04-10T02:55:39.157 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:24 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:39.302 INFO:teuthology.orchestra.run.smithi102.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 278 kB/s | 51 kB 00:00 2024-04-10T02:55:39.445 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:39.445 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:39.446 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.447 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:39.448 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.449 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:39.450 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.451 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.452 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.453 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.454 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:39.515 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 47 k 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:55:39.516 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout:Install 13 Packages 2024-04-10T02:55:39.517 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:39.518 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.2 M 2024-04-10T02:55:39.518 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 26 M 2024-04-10T02:55:39.518 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:39.519 INFO:teuthology.orchestra.run.smithi102.stdout:(5/6): python3-babel-2.9.1-2.el9.noarch.rpm 4.6 MB/s | 6.0 MB 00:01 2024-04-10T02:55:39.627 INFO:teuthology.orchestra.run.smithi102.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 450 kB/s | 548 kB 00:01 2024-04-10T02:55:39.629 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:39.630 INFO:teuthology.orchestra.run.smithi102.stdout:Total 4.2 MB/s | 7.0 MB 00:01 2024-04-10T02:55:39.700 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:39.712 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:39.712 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:39.812 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:39.812 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:39.834 INFO:teuthology.orchestra.run.smithi152.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 258 kB/s | 26 kB 00:00 2024-04-10T02:55:39.859 INFO:teuthology.orchestra.run.smithi152.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 154 kB/s | 19 kB 00:00 2024-04-10T02:55:40.001 INFO:teuthology.orchestra.run.smithi152.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-04-10T02:55:40.027 INFO:teuthology.orchestra.run.smithi152.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 956 kB/s | 159 kB 00:00 2024-04-10T02:55:40.052 INFO:teuthology.orchestra.run.smithi152.stdout:(5/13): ceph-mgr-rook-17.2.7-825.g896bb95d.el9. 150 kB/s | 47 kB 00:00 2024-04-10T02:55:40.085 INFO:teuthology.orchestra.run.smithi152.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 915 kB/s | 54 kB 00:00 2024-04-10T02:55:40.111 INFO:teuthology.orchestra.run.smithi152.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.5 MB/s | 279 kB 00:00 2024-04-10T02:55:40.116 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:40.281 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-10T02:55:40.337 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-10T02:55:40.503 INFO:teuthology.orchestra.run.smithi152.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 71 kB/s | 32 kB 00:00 2024-04-10T02:55:40.536 INFO:teuthology.orchestra.run.smithi152.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 33 kB/s | 15 kB 00:00 2024-04-10T02:55:40.803 INFO:teuthology.orchestra.run.smithi152.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 221 kB/s | 59 kB 00:00 2024-04-10T02:55:40.843 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-10T02:55:40.870 INFO:teuthology.orchestra.run.smithi152.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 267 kB/s | 203 kB 00:00 2024-04-10T02:55:40.938 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-10T02:55:40.995 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-10T02:55:41.020 INFO:teuthology.orchestra.run.smithi152.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 414 kB/s | 90 kB 00:00 2024-04-10T02:55:41.024 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 6/6 2024-04-10T02:55:41.843 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 6/6 2024-04-10T02:55:41.843 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/6 2024-04-10T02:55:41.843 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-10T02:55:41.843 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-10T02:55:41.844 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-10T02:55:41.844 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-10T02:55:41.929 INFO:teuthology.orchestra.run.smithi152.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 733 kB/s | 1.0 MB 00:01 2024-04-10T02:55:41.932 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:41.932 INFO:teuthology.orchestra.run.smithi152.stdout:Total 931 kB/s | 2.2 MB 00:02 2024-04-10T02:55:42.004 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:42.019 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:42.019 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:42.115 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:42.115 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:42.164 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:42.165 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:42.438 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install ceph-fuse 2024-04-10T02:55:42.638 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:42.820 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-10T02:55:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-10T02:55:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-10T02:55:42.981 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-10T02:55:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-10T02:55:43.042 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:31 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-10T02:55:43.239 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-10T02:55:43.301 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.331 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.332 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:43.333 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:43.334 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.335 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.336 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.337 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.338 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.339 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.340 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:43.400 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repository Size 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-fuse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 813 k 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T02:55:43.401 INFO:teuthology.orchestra.run.smithi102.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout:Install 2 Packages 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 893 k 2024-04-10T02:55:43.402 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 2.6 M 2024-04-10T02:55:43.403 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:43.546 INFO:teuthology.orchestra.run.smithi102.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 747 kB/s | 80 kB 00:00 2024-04-10T02:55:44.072 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-10T02:55:44.112 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-10T02:55:44.146 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-10T02:55:44.188 INFO:teuthology.orchestra.run.smithi102.stdout:(2/2): ceph-fuse-17.2.7-825.g896bb95d.el9.x86_6 1.1 MB/s | 813 kB 00:00 2024-04-10T02:55:44.188 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:44.189 INFO:teuthology.orchestra.run.smithi102.stdout:Total 1.1 MB/s | 893 kB 00:00 2024-04-10T02:55:44.192 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-10T02:55:44.202 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:44.215 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 13/13 2024-04-10T02:55:44.226 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:44.226 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:44.305 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:44.306 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:44.496 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:44.625 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-10T02:55:44.687 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2/2 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 13/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 1/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-10T02:55:45.035 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-10T02:55:45.036 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-10T02:55:45.364 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2/2 2024-04-10T02:55:45.365 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/2 2024-04-10T02:55:45.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-10T02:55:45.400 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-10T02:55:45.401 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-10T02:55:45.401 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-10T02:55:45.401 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:45.401 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:45.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-cephadm 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout: ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:45.732 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:46.033 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install librados-devel 2024-04-10T02:55:46.244 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:31 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:46.532 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.533 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.534 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:46.535 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:46.536 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.537 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.538 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.539 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.540 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.602 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm noarch 2:17.2.7-825.g896bb95d.el9 ceph-noarch 123 k 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-10T02:55:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 7.0 M 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 33 M 2024-04-10T02:55:46.604 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:46.638 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:34 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:46.927 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:46.927 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:46.927 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:46.927 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.927 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:46.928 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:46.929 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:46.930 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.931 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.932 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.933 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:46.934 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.935 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:46.995 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:46.996 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repo Size 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout: librados-devel x86_64 2:17.2.7-825.g896bb95d.el9 ceph 126 k 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 126 k 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 448 k 2024-04-10T02:55:46.997 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:47.188 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 848 kB/s | 248 kB 00:00 2024-04-10T02:55:47.454 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): ceph-mgr-cephadm-17.2.7-825.g896bb95d.el 221 kB/s | 123 kB 00:00 2024-04-10T02:55:47.584 INFO:teuthology.orchestra.run.smithi102.stdout:librados-devel-17.2.7-825.g896bb95d.el9.x86_64. 215 kB/s | 126 kB 00:00 2024-04-10T02:55:47.584 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:47.584 INFO:teuthology.orchestra.run.smithi102.stdout:Total 215 kB/s | 126 kB 00:00 2024-04-10T02:55:47.584 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:47.590 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:47.590 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:47.596 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 8.5 MB/s | 6.0 MB 00:00 2024-04-10T02:55:47.650 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:47.650 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:47.806 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:47.924 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:47.947 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 722 kB/s | 548 kB 00:00 2024-04-10T02:55:47.972 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 112 kB/s | 58 kB 00:00 2024-04-10T02:55:48.039 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 122 kB/s | 51 kB 00:00 2024-04-10T02:55:48.041 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:48.041 INFO:teuthology.orchestra.run.smithi152.stdout:Total 4.8 MB/s | 7.0 MB 00:01 2024-04-10T02:55:48.112 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:48.123 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:48.123 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:48.224 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:48.226 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:48.339 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:48.533 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout: librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:48.646 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:48.702 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-10T02:55:48.750 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-10T02:55:48.900 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install libcephfs2 2024-04-10T02:55:49.264 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-10T02:55:49.353 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-10T02:55:49.407 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-10T02:55:49.438 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 6/6 2024-04-10T02:55:49.502 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:37 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:49.795 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:49.796 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.797 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.798 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.799 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.800 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.801 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.802 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.803 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:49.804 INFO:teuthology.orchestra.run.smithi102.stdout:Package libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:49.864 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:49.865 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:49.865 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:49.942 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install libcephfs-devel 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 6/6 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 1/6 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-10T02:55:50.240 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-10T02:55:50.531 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:38 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:50.568 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-10T02:55:50.568 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:50.568 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:50.568 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm-2:17.2.7-825.g896bb95d.el9.noarch 2024-04-10T02:55:50.568 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:50.569 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:50.818 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.819 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.820 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:50.821 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.822 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:50.823 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.824 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.825 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.826 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.827 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:50.827 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.827 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.827 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.827 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:50.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-fuse 2024-04-10T02:55:50.887 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:50.888 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:50.888 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repo Size 2024-04-10T02:55:50.888 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:50.888 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout: libcephfs-devel x86_64 2:17.2.7-825.g896bb95d.el9 ceph 25 k 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 25 k 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 110 k 2024-04-10T02:55:50.889 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:51.046 INFO:teuthology.orchestra.run.smithi102.stdout:libcephfs-devel-17.2.7-825.g896bb95d.el9.x86_64 159 kB/s | 25 kB 00:00 2024-04-10T02:55:51.047 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:51.047 INFO:teuthology.orchestra.run.smithi102.stdout:Total 158 kB/s | 25 kB 00:00 2024-04-10T02:55:51.047 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:51.050 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:51.050 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:51.051 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:51.052 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:51.127 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:51.235 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:51.430 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:36 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:51.585 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:51.721 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.722 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:51.723 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.724 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:51.725 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:51.726 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.727 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.728 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.729 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.730 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:51.791 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 813 k 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout:Install 2 Packages 2024-04-10T02:55:51.792 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:51.793 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 893 k 2024-04-10T02:55:51.793 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 2.6 M 2024-04-10T02:55:51.793 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout: libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:51.862 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:51.967 INFO:teuthology.orchestra.run.smithi152.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 580 kB/s | 80 kB 00:00 2024-04-10T02:55:52.056 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install librados2 2024-04-10T02:55:52.653 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:40 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:52.947 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:52.947 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:52.947 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:52.947 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.947 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:52.948 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.949 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.950 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:52.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.951 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:52.951 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:52.953 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:52.954 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.955 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:52.956 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:52.957 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:52.958 INFO:teuthology.orchestra.run.smithi102.stdout:Package librados2-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:53.016 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:53.017 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:53.017 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:53.099 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install librbd1 2024-04-10T02:55:53.350 INFO:teuthology.orchestra.run.smithi152.stdout:(2/2): ceph-fuse-17.2.7-825.g896bb95d.el9.x86_6 534 kB/s | 813 kB 00:01 2024-04-10T02:55:53.350 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:53.351 INFO:teuthology.orchestra.run.smithi152.stdout:Total 573 kB/s | 893 kB 00:01 2024-04-10T02:55:53.365 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:53.390 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:53.390 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:53.470 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:53.470 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:53.653 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:53.696 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:41 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:53.787 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-10T02:55:53.850 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2/2 2024-04-10T02:55:53.984 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:53.984 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:53.984 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.985 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:53.986 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.987 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:53.988 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.989 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.990 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.991 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:53.992 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.993 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.993 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.993 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:53.993 INFO:teuthology.orchestra.run.smithi102.stdout:Package librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:54.053 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:54.055 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:54.055 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:54.128 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install python3-rados 2024-04-10T02:55:54.512 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2/2 2024-04-10T02:55:54.512 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/2 2024-04-10T02:55:54.715 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:42 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:54.835 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.003 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:55.004 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.005 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.006 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.007 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.008 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.009 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.010 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.011 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:55.012 INFO:teuthology.orchestra.run.smithi102.stdout:Package python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:55.071 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:55.087 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:55.087 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:55.141 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados-devel 2024-04-10T02:55:55.149 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install python3-rgw 2024-04-10T02:55:55.741 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:43 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:55.763 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:40 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:56.028 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:56.028 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:56.028 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:56.028 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:56.029 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:56.030 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:56.031 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:56.032 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.033 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.034 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.035 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.036 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.037 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.037 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.037 INFO:teuthology.orchestra.run.smithi102.stdout:Package python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:56.046 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:56.047 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:56.048 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.049 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.050 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.051 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.052 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.053 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.054 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.055 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:56.101 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:56.101 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:56.101 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:56.122 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:56.122 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel x86_64 2:17.2.7-825.g896bb95d.el9 ceph 126 k 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 126 k 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 448 k 2024-04-10T02:55:56.123 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:56.177 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install python3-cephfs 2024-04-10T02:55:56.337 INFO:teuthology.orchestra.run.smithi152.stdout:librados-devel-17.2.7-825.g896bb95d.el9.x86_64. 577 kB/s | 126 kB 00:00 2024-04-10T02:55:56.337 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:56.340 INFO:teuthology.orchestra.run.smithi152.stdout:Total 573 kB/s | 126 kB 00:00 2024-04-10T02:55:56.340 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:56.343 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:56.343 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:56.403 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:56.403 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:55:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:56.776 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:44 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:57.065 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.068 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.069 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:57.070 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:57.071 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.072 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.073 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:57.074 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:57.075 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.076 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:57.077 INFO:teuthology.orchestra.run.smithi102.stdout:Package python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:57.136 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:57.137 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:57.137 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:57.219 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install python3-rbd 2024-04-10T02:55:57.375 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:57.375 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:57.375 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:55:57.376 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:55:57.376 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:57.376 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:57.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs2 2024-04-10T02:55:57.808 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:45 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:58.102 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:58.103 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.104 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.105 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.106 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:58.107 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.108 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:58.109 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:58.110 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.111 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.111 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.111 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.111 INFO:teuthology.orchestra.run.smithi102.stdout:Package python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:58.171 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:58.172 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-04-10T02:55:58.172 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:55:58.252 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install rbd-fuse 2024-04-10T02:55:58.256 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:43 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:58.548 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:58.549 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:58.550 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.551 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:58.552 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.553 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.554 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:58.555 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:58.556 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.557 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:58.558 INFO:teuthology.orchestra.run.smithi152.stdout:Package libcephfs2-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:55:58.618 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:58.619 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:55:58.619 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:55:58.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs-devel 2024-04-10T02:55:58.843 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:46 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:55:59.130 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:55:59.130 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:55:59.131 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:55:59.132 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.133 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.134 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.135 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.136 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:59.137 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.138 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:59.139 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.139 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.139 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.139 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.201 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:55:59.201 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-fuse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 82 k 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 82 k 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 222 k 2024-04-10T02:55:59.202 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:55:59.296 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:44 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:55:59.515 INFO:teuthology.orchestra.run.smithi102.stdout:rbd-fuse-17.2.7-825.g896bb95d.el9.x86_64.rpm 261 kB/s | 82 kB 00:00 2024-04-10T02:55:59.516 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:59.516 INFO:teuthology.orchestra.run.smithi102.stdout:Total 260 kB/s | 82 kB 00:00 2024-04-10T02:55:59.516 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:55:59.523 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:55:59.523 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:55:59.582 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:55:59.582 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.585 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:55:59.586 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.587 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:55:59.588 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.589 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.590 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:55:59.591 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.592 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.593 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.593 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:55:59.654 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel x86_64 2:17.2.7-825.g896bb95d.el9 ceph 25 k 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 25 k 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 110 k 2024-04-10T02:55:59.655 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:55:59.719 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:55:59.829 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:55:59.831 INFO:teuthology.orchestra.run.smithi152.stdout:libcephfs-devel-17.2.7-825.g896bb95d.el9.x86_64 142 kB/s | 25 kB 00:00 2024-04-10T02:55:59.831 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:55:59.832 INFO:teuthology.orchestra.run.smithi152.stdout:Total 141 kB/s | 25 kB 00:00 2024-04-10T02:55:59.832 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:55:59.835 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:55:59.835 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:55:59.836 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:55:59.837 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:55:59.925 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:56:00.028 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:00.215 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:00.430 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:00.519 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:56:00.705 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:00.706 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:00.706 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:56:00.706 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:00.706 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:00.706 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:00.791 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install rbd-mirror 2024-04-10T02:56:00.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados2 2024-04-10T02:56:01.395 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:49 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:56:01.489 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:46 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:56:01.697 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.698 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:56:01.699 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:56:01.700 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.701 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.702 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:01.703 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.704 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.705 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.769 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:56:01.769 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:01.769 INFO:teuthology.orchestra.run.smithi102.stdout: Package Arch Version Repo Size 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-mirror x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.9 M 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 2.9 M 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 12 M 2024-04-10T02:56:01.770 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.784 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:01.785 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:01.786 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.787 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:01.788 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.789 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.790 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.791 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.792 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:01.793 INFO:teuthology.orchestra.run.smithi152.stdout:Package librados2-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:01.854 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:01.855 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:01.855 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:01.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librbd1 2024-04-10T02:56:02.542 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:47 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:02.652 INFO:teuthology.orchestra.run.smithi102.stdout:rbd-mirror-17.2.7-825.g896bb95d.el9.x86_64.rpm 3.3 MB/s | 2.9 MB 00:00 2024-04-10T02:56:02.654 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:02.654 INFO:teuthology.orchestra.run.smithi102.stdout:Total 3.3 MB/s | 2.9 MB 00:00 2024-04-10T02:56:02.654 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:56:02.670 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:56:02.670 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:56:02.763 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:56:02.763 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:02.831 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.832 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:02.833 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:02.834 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.835 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:02.836 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.837 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.838 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.839 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:02.840 INFO:teuthology.orchestra.run.smithi152.stdout:Package librbd1-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:02.901 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:02.902 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:02.902 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:02.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rados 2024-04-10T02:56:03.172 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:56:03.237 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:03.262 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:03.262 INFO:teuthology.orchestra.run.smithi102.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:56:03.263 INFO:teuthology.orchestra.run.smithi102.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-10T02:56:03.263 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-10T02:56:03.263 INFO:teuthology.orchestra.run.smithi102.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-10T02:56:03.263 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:03.591 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:48 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:03.882 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.883 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:03.884 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:03.885 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.886 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:03.887 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.888 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.889 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.890 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.891 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.892 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:03.892 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rados-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:03.951 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:03.952 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:03.952 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:04.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rgw 2024-04-10T02:56:04.377 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:04.377 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:04.377 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:56:04.378 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:04.378 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:04.378 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:56:04.637 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum -y install rbd-nbd 2024-04-10T02:56:04.637 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:49 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:04.923 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:04.924 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.925 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:04.926 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:04.927 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:04.928 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.929 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:04.930 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.931 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:04.932 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.932 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.932 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.932 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:04.932 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rgw-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:04.992 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:04.993 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:04.993 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:05.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-cephfs 2024-04-10T02:56:05.248 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:01:53 ago on Wed 10 Apr 2024 02:54:12 AM UTC. 2024-04-10T02:56:05.536 INFO:teuthology.orchestra.run.smithi102.stderr:Modular dependency problems: 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 1: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 2: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 3: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 4: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 5: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 6: conflicting requests 2024-04-10T02:56:05.537 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 7: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 8: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 9: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 10: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 11: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 12: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 13: conflicting requests 2024-04-10T02:56:05.538 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 14: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 15: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 16: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 17: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 18: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 19: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 20: conflicting requests 2024-04-10T02:56:05.539 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 21: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 22: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 23: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 24: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 25: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 26: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 27: conflicting requests 2024-04-10T02:56:05.540 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 28: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 29: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 30: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 31: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 32: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 33: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 34: conflicting requests 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.541 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 35: conflicting requests 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 36: conflicting requests 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 37: conflicting requests 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 39: conflicting requests 2024-04-10T02:56:05.542 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.543 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.544 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.545 INFO:teuthology.orchestra.run.smithi102.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.606 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-nbd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 165 k 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:Install 1 Package 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 165 k 2024-04-10T02:56:05.607 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 477 k 2024-04-10T02:56:05.608 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T02:56:05.684 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:50 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:05.950 INFO:teuthology.orchestra.run.smithi102.stdout:rbd-nbd-17.2.7-825.g896bb95d.el9.x86_64.rpm 484 kB/s | 165 kB 00:00 2024-04-10T02:56:05.950 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:05.950 INFO:teuthology.orchestra.run.smithi102.stdout:Total 482 kB/s | 165 kB 00:00 2024-04-10T02:56:05.950 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T02:56:05.958 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T02:56:05.958 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T02:56:05.980 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:05.980 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:05.980 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.981 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:05.982 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:05.983 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:05.984 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.985 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.986 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:05.987 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.988 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:05.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.989 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:05.989 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-cephfs-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:06.018 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T02:56:06.018 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T02:56:06.050 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:06.051 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:06.051 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:06.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rbd 2024-04-10T02:56:06.163 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T02:56:06.265 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:06.698 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:06.772 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:51 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout: rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:07.005 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T02:56:07.066 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:07.066 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:07.066 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:07.067 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:07.068 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:07.069 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.070 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.071 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.072 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.073 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.074 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:07.075 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rbd-2:17.2.7-825.g896bb95d.el9.x86_64 is already installed. 2024-04-10T02:56:07.135 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:07.136 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-10T02:56:07.136 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:07.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-fuse 2024-04-10T02:56:07.251 DEBUG:teuthology.parallel:result is None 2024-04-10T02:56:07.832 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:52 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:08.122 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.123 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.124 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:08.125 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:08.126 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:08.127 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.128 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.129 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.130 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse x86_64 2:17.2.7-825.g896bb95d.el9 ceph 82 k 2024-04-10T02:56:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 222 k 2024-04-10T02:56:08.193 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:56:08.509 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-fuse-17.2.7-825.g896bb95d.el9.x86_64.rpm 259 kB/s | 82 kB 00:00 2024-04-10T02:56:08.510 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:08.510 INFO:teuthology.orchestra.run.smithi152.stdout:Total 258 kB/s | 82 kB 00:00 2024-04-10T02:56:08.510 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:56:08.517 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:56:08.517 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:56:08.575 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:56:08.575 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:56:08.736 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:56:08.823 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:09.261 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:09.563 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:09.563 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:09.563 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:56:09.563 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:09.563 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:09.564 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:09.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-mirror 2024-04-10T02:56:10.398 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:55 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:10.696 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:10.696 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:10.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:10.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:10.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:10.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:10.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:10.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.704 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.705 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:10.765 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:10.766 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror x86_64 2:17.2.7-825.g896bb95d.el9 ceph 2.9 M 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.9 M 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 12 M 2024-04-10T02:56:10.767 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:56:11.811 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-mirror-17.2.7-825.g896bb95d.el9.x86_64.rpm 2.8 MB/s | 2.9 MB 00:01 2024-04-10T02:56:11.812 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:11.812 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.8 MB/s | 2.9 MB 00:01 2024-04-10T02:56:11.812 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:56:11.822 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:56:11.822 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:56:11.898 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:56:11.899 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:56:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:56:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-10T02:56:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:13.533 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:13.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-nbd 2024-04-10T02:56:14.403 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:59 ago on Wed 10 Apr 2024 02:54:15 AM UTC. 2024-04-10T02:56:14.694 INFO:teuthology.orchestra.run.smithi152.stderr:Modular dependency problems: 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 1: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 2: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 3: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 4: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 5: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 6: conflicting requests 2024-04-10T02:56:14.695 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 7: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 8: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 9: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 10: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 11: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 12: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 13: conflicting requests 2024-04-10T02:56:14.696 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 14: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 15: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 16: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 17: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 18: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 19: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 20: conflicting requests 2024-04-10T02:56:14.697 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 21: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 22: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 23: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 24: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 25: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 26: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 27: conflicting requests 2024-04-10T02:56:14.698 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 28: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 29: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 30: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 31: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 32: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 33: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 34: conflicting requests 2024-04-10T02:56:14.699 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 35: conflicting requests 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 36: conflicting requests 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 37: conflicting requests 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 38: cannot install the best candidate for the job 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 39: conflicting requests 2024-04-10T02:56:14.700 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 40: cannot install the best candidate for the job 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 41: cannot install the best candidate for the job 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.701 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 42: cannot install the best candidate for the job 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 43: cannot install the best candidate for the job 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 44: cannot install the best candidate for the job 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.702 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: Problem 45: cannot install the best candidate for the job 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.703 INFO:teuthology.orchestra.run.smithi152.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-10T02:56:14.764 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-10T02:56:14.764 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:14.764 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-10T02:56:14.764 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd x86_64 2:17.2.7-825.g896bb95d.el9 ceph 165 k 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:14.765 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 165 k 2024-04-10T02:56:14.766 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 477 k 2024-04-10T02:56:14.766 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-10T02:56:15.113 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-nbd-17.2.7-825.g896bb95d.el9.x86_64.rpm 476 kB/s | 165 kB 00:00 2024-04-10T02:56:15.114 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:15.114 INFO:teuthology.orchestra.run.smithi152.stdout:Total 474 kB/s | 165 kB 00:00 2024-04-10T02:56:15.114 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-10T02:56:15.122 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-10T02:56:15.122 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-10T02:56:15.180 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-10T02:56:15.181 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-10T02:56:15.334 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-10T02:56:15.437 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:15.850 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 1/1 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd-2:17.2.7-825.g896bb95d.el9.x86_64 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:56:16.187 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-10T02:56:16.449 DEBUG:teuthology.parallel:result is None 2024-04-10T02:56:16.449 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:56:16.449 INFO:teuthology.packaging:ref: None 2024-04-10T02:56:16.449 INFO:teuthology.packaging:tag: None 2024-04-10T02:56:16.449 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:56:16.449 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:56:16.449 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-10T02:56:16.672 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-10T02:56:16.696 INFO:teuthology.orchestra.run.smithi102.stdout:17.2.7-825.g896bb95d.el9 2024-04-10T02:56:16.697 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-825.g896bb95d.el9 2024-04-10T02:56:16.697 INFO:teuthology.task.install:The correct ceph version 17.2.7-825.g896bb95d is installed. 2024-04-10T02:56:16.699 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T02:56:16.699 INFO:teuthology.packaging:ref: None 2024-04-10T02:56:16.699 INFO:teuthology.packaging:tag: None 2024-04-10T02:56:16.699 INFO:teuthology.packaging:branch: quincy 2024-04-10T02:56:16.699 INFO:teuthology.packaging:sha1: 3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T02:56:16.699 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-10T02:56:16.999 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-10T02:56:17.019 INFO:teuthology.orchestra.run.smithi152.stdout:17.2.7-825.g896bb95d.el9 2024-04-10T02:56:17.020 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-825.g896bb95d.el9 2024-04-10T02:56:17.020 INFO:teuthology.task.install:The correct ceph version 17.2.7-825.g896bb95d is installed. 2024-04-10T02:56:17.021 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-10T02:56:17.022 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:56:17.022 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-10T02:56:17.054 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:56:17.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-10T02:56:17.087 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-10T02:56:17.087 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:56:17.087 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/usr/bin/daemon-helper 2024-04-10T02:56:17.125 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-10T02:56:17.195 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:56:17.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/daemon-helper 2024-04-10T02:56:17.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-10T02:56:17.289 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-10T02:56:17.290 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:56:17.290 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-10T02:56:17.320 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-10T02:56:17.390 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:56:17.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-10T02:56:17.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-10T02:56:17.487 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-10T02:56:17.487 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:56:17.487 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/usr/bin/stdin-killer 2024-04-10T02:56:17.516 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-10T02:56:17.585 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:56:17.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/stdin-killer 2024-04-10T02:56:17.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-10T02:56:17.679 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-10T02:56:17.776 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '3a866975a60f6dc859246f6f19fd8ddb4db2cc2e'} 2024-04-10T02:56:17.776 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-10T02:56:17.777 INFO:tasks.cephadm:Cluster fsid is e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:56:17.777 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-10T02:56:17.777 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.102', 'mon.c': '[v2:172.21.15.102:3301,v1:172.21.15.102:6790]', 'mon.b': '172.21.15.152'} 2024-04-10T02:56:17.777 INFO:tasks.cephadm:First mon is mon.a on smithi102 2024-04-10T02:56:17.777 INFO:tasks.cephadm:First mgr is y 2024-04-10T02:56:17.777 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-10T02:56:17.777 DEBUG:teuthology.orchestra.run.smithi102:> sudo hostname $(hostname -s) 2024-04-10T02:56:17.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo hostname $(hostname -s) 2024-04-10T02:56:17.834 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-10T02:56:17.835 DEBUG:teuthology.orchestra.run.smithi102:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.062 INFO:teuthology.orchestra.run.smithi102.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 10 02:56 /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.062 DEBUG:teuthology.orchestra.run.smithi152:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.234 INFO:teuthology.orchestra.run.smithi152.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 10 02:56 /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.234 DEBUG:teuthology.orchestra.run.smithi102:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.252 DEBUG:teuthology.orchestra.run.smithi152:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-10T02:56:18.281 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-10T02:56:18.281 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-10T02:56:18.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-10T02:56:19.184 INFO:teuthology.orchestra.run.smithi102.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-10T02:56:19.198 INFO:teuthology.orchestra.run.smithi152.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-10T02:56:47.993 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph_version": "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)", 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout: "image_id": "21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241", 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout: "repo_digests": [ 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe" 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-04-10T02:56:47.994 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-10T02:56:51.651 INFO:teuthology.orchestra.run.smithi102.stdout:{ 2024-04-10T02:56:51.651 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph_version": "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)", 2024-04-10T02:56:51.651 INFO:teuthology.orchestra.run.smithi102.stdout: "image_id": "21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241", 2024-04-10T02:56:51.651 INFO:teuthology.orchestra.run.smithi102.stdout: "repo_digests": [ 2024-04-10T02:56:51.652 INFO:teuthology.orchestra.run.smithi102.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe" 2024-04-10T02:56:51.652 INFO:teuthology.orchestra.run.smithi102.stdout: ] 2024-04-10T02:56:51.652 INFO:teuthology.orchestra.run.smithi102.stdout:} 2024-04-10T02:56:51.671 DEBUG:teuthology.orchestra.run.smithi102:> sudo mkdir -p /etc/ceph 2024-04-10T02:56:51.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo mkdir -p /etc/ceph 2024-04-10T02:56:51.740 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 777 /etc/ceph 2024-04-10T02:56:51.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 777 /etc/ceph 2024-04-10T02:56:51.811 INFO:tasks.cephadm:Writing seed config... 2024-04-10T02:56:51.812 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-10T02:56:51.813 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-10T02:56:51.814 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:56:51.814 DEBUG:teuthology.orchestra.run.smithi102:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-10T02:56:51.828 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-10T02:56:51.829 DEBUG:teuthology.orchestra.run.smithi102:mon.a> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service 2024-04-10T02:56:51.875 DEBUG:teuthology.orchestra.run.smithi102:mgr.y> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service 2024-04-10T02:56:51.917 INFO:tasks.cephadm:Bootstrapping... 2024-04-10T02:56:51.917 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.102 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-10T02:56:52.122 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T02:56:52.122 INFO:teuthology.orchestra.run.smithi102.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.102', '--skip-admin-label'] 2024-04-10T02:56:52.147 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-10T02:56:52.147 INFO:teuthology.orchestra.run.smithi102.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-10T02:56:52.147 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying podman|docker is present... 2024-04-10T02:56:52.170 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-10T02:56:52.170 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying lvm2 is present... 2024-04-10T02:56:52.170 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying time synchronization is in place... 2024-04-10T02:56:52.178 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-10T02:56:52.178 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-10T02:56:52.184 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-10T02:56:52.184 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:56:52.191 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout enabled 2024-04-10T02:56:52.197 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout active 2024-04-10T02:56:52.197 INFO:teuthology.orchestra.run.smithi102.stdout:Unit chronyd.service is enabled and running 2024-04-10T02:56:52.197 INFO:teuthology.orchestra.run.smithi102.stdout:Repeating the final host check... 2024-04-10T02:56:52.220 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-10T02:56:52.220 INFO:teuthology.orchestra.run.smithi102.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-10T02:56:52.220 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl is present 2024-04-10T02:56:52.221 INFO:teuthology.orchestra.run.smithi102.stdout:lvcreate is present 2024-04-10T02:56:52.227 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-10T02:56:52.227 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-10T02:56:52.233 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-10T02:56:52.233 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:56:52.240 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout enabled 2024-04-10T02:56:52.246 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout active 2024-04-10T02:56:52.246 INFO:teuthology.orchestra.run.smithi102.stdout:Unit chronyd.service is enabled and running 2024-04-10T02:56:52.246 INFO:teuthology.orchestra.run.smithi102.stdout:Host looks OK 2024-04-10T02:56:52.246 INFO:teuthology.orchestra.run.smithi102.stdout:Cluster fsid: e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:56:52.247 INFO:teuthology.orchestra.run.smithi102.stdout:Acquiring lock 140485842818144 on /run/cephadm/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.lock 2024-04-10T02:56:52.247 INFO:teuthology.orchestra.run.smithi102.stdout:Lock 140485842818144 acquired on /run/cephadm/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.lock 2024-04-10T02:56:52.247 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying IP 172.21.15.102 port 3300 ... 2024-04-10T02:56:52.247 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying IP 172.21.15.102 port 6789 ... 2024-04-10T02:56:52.247 INFO:teuthology.orchestra.run.smithi102.stdout:Base mon IP(s) is [172.21.15.102:3300, 172.21.15.102:6789], mon addrv is [v2:172.21.15.102:3300,v1:172.21.15.102:6789] 2024-04-10T02:56:52.250 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.102 metric 100 2024-04-10T02:56:52.250 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.102 metric 100 2024-04-10T02:56:52.253 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-10T02:56:52.253 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-10T02:56:52.253 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1794sec hoplimit 64 pref medium 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d769/64 scope link 2024-04-10T02:56:52.256 INFO:teuthology.orchestra.run.smithi102.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-10T02:56:52.257 INFO:teuthology.orchestra.run.smithi102.stdout:Mon IP `172.21.15.102` is in CIDR network `172.21.0.0/20` 2024-04-10T02:56:52.257 INFO:teuthology.orchestra.run.smithi102.stdout:Mon IP `172.21.15.102` is in CIDR network `172.21.0.0/20` 2024-04-10T02:56:52.257 INFO:teuthology.orchestra.run.smithi102.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-10T02:56:52.257 INFO:teuthology.orchestra.run.smithi102.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-10T02:56:52.258 INFO:teuthology.orchestra.run.smithi102.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-10T02:56:52.717 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stdout 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 2024-04-10T02:56:52.717 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-10T02:56:52.718 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Getting image source signatures 2024-04-10T02:56:52.718 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Copying blob sha256:6a38c9a31ea4c6c1e3d079c8edc2f3b418b00f3ce1d3a48ffe92dcb2ef7b4e51 2024-04-10T02:56:52.718 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Copying blob sha256:9793b5b5582798bde394f4383d142e9f248a25e4ebfd72ee05ea9ec6251a877d 2024-04-10T02:56:52.718 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Copying config sha256:21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 2024-04-10T02:56:52.718 INFO:teuthology.orchestra.run.smithi102.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-10T02:56:53.639 INFO:teuthology.orchestra.run.smithi102.stdout:ceph: stdout ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable) 2024-04-10T02:56:53.639 INFO:teuthology.orchestra.run.smithi102.stdout:Ceph version: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable) 2024-04-10T02:56:53.640 INFO:teuthology.orchestra.run.smithi102.stdout:Extracting ceph user uid/gid from container image... 2024-04-10T02:56:54.586 INFO:teuthology.orchestra.run.smithi102.stdout:stat: stdout 167 167 2024-04-10T02:56:54.586 INFO:teuthology.orchestra.run.smithi102.stdout:Creating initial keys... 2024-04-10T02:56:55.467 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph-authtool: stdout AQD2/xVmgV3YNBAAkUURLxsNJpdYnCTw/dWkJA== 2024-04-10T02:56:56.389 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph-authtool: stdout AQD3/xVmevEvLxAAIDV4eRZArdkPnZ2GlN0stQ== 2024-04-10T02:56:57.395 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph-authtool: stdout AQD4/xVmsM+ZKRAAZBbU3gKu7ZCpTl7uh55pxA== 2024-04-10T02:56:57.396 INFO:teuthology.orchestra.run.smithi102.stdout:Creating initial monmap... 2024-04-10T02:56:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-10T02:56:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-10T02:56:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:56:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-10T02:56:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:monmaptool for a [v2:172.21.15.102:3300,v1:172.21.15.102:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-10T02:56:58.286 INFO:teuthology.orchestra.run.smithi102.stdout:setting min_mon_release = octopus 2024-04-10T02:56:58.286 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: set fsid to e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:56:58.286 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-10T02:56:58.286 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:56:58.286 INFO:teuthology.orchestra.run.smithi102.stdout:Creating mon... 2024-04-10T02:56:59.183 INFO:teuthology.orchestra.run.smithi102.stdout:create mon.a on 2024-04-10T02:56:59.382 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-10T02:56:59.530 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-10T02:56:59.687 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target → /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target. 2024-04-10T02:56:59.687 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target → /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target. 2024-04-10T02:56:59.873 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a 2024-04-10T02:56:59.874 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service: Unit ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service not loaded. 2024-04-10T02:57:00.036 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target.wants/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@.service. 2024-04-10T02:57:00.623 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-10T02:57:00.623 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout disabled 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:firewalld.service is not enabled 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:Not possible to enable service . firewalld.service is not available 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mon to start... 2024-04-10T02:57:00.631 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mon... 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout cluster: 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout id: e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout services: 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.643228s) 2024-04-10T02:57:01.704 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout data: 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout pgs: 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:mon is available 2024-04-10T02:57:01.705 INFO:teuthology.orchestra.run.smithi102.stdout:Assimilating anything we can from ceph.conf... 2024-04-10T02:57:02.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:02 smithi102 ceph-mon[28929]: from='client.? 172.21.15.102:0/3650545592' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [global] 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout fsid = e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.102:3300,v1:172.21.15.102:6789] 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:02.802 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [osd] 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-10T02:57:02.803 INFO:teuthology.orchestra.run.smithi102.stdout:Generating new minimal ceph.conf... 2024-04-10T02:57:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:03 smithi102 ceph-mon[28929]: from='client.? 172.21.15.102:0/987840577' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:57:03.816 INFO:teuthology.orchestra.run.smithi102.stdout:Restarting the monitor... 2024-04-10T02:57:04.083 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:03 smithi102 systemd[1]: Stopping Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T02:57:04.350 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[28925]: 2024-04-10T02:57:04.083+0000 7f6c6c46b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T02:57:04.350 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[28925]: 2024-04-10T02:57:04.083+0000 7f6c6c46b700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-10T02:57:04.350 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 podman[29174]: 2024-04-10 02:57:04.135576222 +0000 UTC m=+0.158678312 container died 968e54770c466cae0efccd701aaeb62a34bc64f7f72b68fbf5ebc91b75082245 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T02:57:04.608 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 podman[29174]: 2024-04-10 02:57:04.450561722 +0000 UTC m=+0.473663820 container cleanup 968e54770c466cae0efccd701aaeb62a34bc64f7f72b68fbf5ebc91b75082245 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, ceph=True, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-10T02:57:04.608 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 bash[29174]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a 2024-04-10T02:57:05.017 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 podman[29188]: 2024-04-10 02:57:04.609912839 +0000 UTC m=+0.469594567 container remove 968e54770c466cae0efccd701aaeb62a34bc64f7f72b68fbf5ebc91b75082245 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, ceph=True, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T02:57:05.017 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-10T02:57:05.017 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 systemd[1]: Stopped Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:57:05.017 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:04 smithi102 systemd[1]: Starting Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T02:57:05.272 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 podman[29282]: 2024-04-10 02:57:05.01900206 +0000 UTC m=+0.105255395 container create 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS) 2024-04-10T02:57:05.272 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 podman[29282]: 2024-04-10 02:57:04.93070645 +0000 UTC m=+0.016959800 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-10T02:57:05.272 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 podman[29282]: 2024-04-10 02:57:05.21526382 +0000 UTC m=+0.301517155 container init 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-10T02:57:05.272 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 podman[29282]: 2024-04-10 02:57:05.218610984 +0000 UTC m=+0.304864328 container start 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: set uid:gid to 167:167 (ceph:ceph) 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable), process ceph-mon, pid 2 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: pidfile_write: ignore empty --pid-file 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: load: jerasure load: lrc 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: RocksDB version: 6.15.5 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Compile date Apr 9 2024 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: DB SUMMARY 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: DB Session ID: 4OA98BT3VFIYQVG87YFS 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: CURRENT file: CURRENT 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: IDENTITY file: IDENTITY 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-10T02:57:05.273 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84243 ; 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.error_if_exists: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.create_if_missing: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.paranoid_checks: 1 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.env: 0x5562b9378a80 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.fs: Posix File System 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.info_log: 0x5562ba670e80 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_file_opening_threads: 16 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.statistics: (nil) 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.use_fsync: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_log_file_size: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-10T02:57:05.274 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.keep_log_file_num: 1000 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.recycle_log_file_num: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_fallocate: 1 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_mmap_reads: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_mmap_writes: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.use_direct_reads: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.create_missing_column_families: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.db_log_dir: 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-10T02:57:05.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.advise_random_on_open: 1 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.db_write_buffer_size: 0 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.write_buffer_manager: 0x5562bb3e0900 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.rate_limiter: (nil) 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.wal_recovery_mode: 2 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.enable_thread_tracking: 0 2024-04-10T02:57:05.276 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.enable_pipelined_write: 0 2024-04-10T02:57:05.277 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.unordered_write: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.row_cache: None 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.wal_filter: None 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_ingest_behind: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.preserve_deletes: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.two_write_queues: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.manual_wal_flush: 0 2024-04-10T02:57:05.278 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.atomic_flush: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.log_readahead_size: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.best_efforts_recovery: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.allow_data_in_errors: 0 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.db_host_id: __hostname__ 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_background_jobs: 2 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_background_compactions: -1 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_subcompactions: 1 2024-04-10T02:57:05.279 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_total_wal_size: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_open_files: -1 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bytes_per_sync: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_readahead_size: 0 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_background_flushes: -1 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Compression algorithms supported: 2024-04-10T02:57:05.280 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kZSTD supported: 0 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kXpressCompression supported: 0 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kLZ4HCCompression supported: 1 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kZlibCompression supported: 1 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kSnappyCompression supported: 1 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kLZ4Compression supported: 1 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: kBZip2Compression supported: 0 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.merge_operator: 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_filter: None 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_filter_factory: None 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.sst_partitioner_factory: None 2024-04-10T02:57:05.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5562ba6c0f58) 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_type: 0 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: data_block_index_type: 0 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_shortening: 1 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: hash_index_allow_collision: 1 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: checksum: 1 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: no_block_cache: 0 2024-04-10T02:57:05.282 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache: 0x5562ba6f3090 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_options: 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: capacity : 536870912 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: num_shard_bits : 4 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: strict_capacity_limit : 0 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_compressed: (nil) 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: persistent_cache: (nil) 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_size: 4096 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_size_deviation: 10 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_restart_interval: 16 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_block_restart_interval: 1 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: metadata_block_size: 4096 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: partition_filters: 0 2024-04-10T02:57:05.283 INFO:journalctl@ceph.mon.a.smithi102.stdout: use_delta_encoding: 1 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: whole_key_filtering: 1 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: verify_compression: 0 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: format_version: 4 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: enable_index_compression: 1 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_align: 0 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.write_buffer_size: 33554432 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_write_buffer_number: 2 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression: NoCompression 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression: Disabled 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.prefix_extractor: nullptr 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-10T02:57:05.284 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.num_levels: 7 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.level: 32767 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.strategy: 0 2024-04-10T02:57:05.285 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compression_opts.enabled: false 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.target_file_size_base: 67108864 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-10T02:57:05.286 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.arena_block_size: 4194304 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.disable_auto_compactions: 0 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-10T02:57:05.287 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.table_properties_collectors: 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.inplace_update_support: 0 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.bloom_locality: 0 2024-04-10T02:57:05.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.max_successive_merges: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.paranoid_file_checks: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.force_consistency_checks: 1 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.report_bg_io_stats: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.ttl: 2592000 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.enable_blob_files: false 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.min_blob_size: 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.blob_file_size: 268435456 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-10T02:57:05.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717825244305, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717825245045, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80675, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78936, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73145, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712717825, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a6469ba1-df75-42dc-92ce-cb784a6c871b", "db_session_id": "4OA98BT3VFIYQVG87YFS"}} 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717825245395, "job": 1, "event": "recovery_finished"} 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5562ba687180 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: DB pointer 0x5562ba75c000 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-10T02:57:05.290 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** DB Stats ** 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: L0 2/0 80.54 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Sum 2/0 80.54 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative compaction: 0.00 GB write, 26.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval compaction: 0.00 GB write, 26.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:57:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: L0 2/0 80.54 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Sum 2/0 80.54 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T02:57:05.293 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative compaction: 0.00 GB write, 25.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: starting mon.a rank 0 at public addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] at bind addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???) e1 preinit fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).mds e1 new map 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).mds e1 print_map 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: e1 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: legacy client fscid: -1 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T02:57:05.294 INFO:journalctl@ceph.mon.a.smithi102.stdout: No filesystems configured 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-10T02:57:05.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 bash[29282]: 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 2024-04-10T02:57:05.295 INFO:teuthology.orchestra.run.smithi102.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-10T02:57:05.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 systemd[1]: Started Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:57:05.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-10T02:57:05.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: monmap e1: 1 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:57:05.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: fsmap 2024-04-10T02:57:05.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: osdmap e1: 0 total, 0 up, 0 in 2024-04-10T02:57:05.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:05 smithi102 ceph-mon[29301]: mgrmap e1: no daemons active 2024-04-10T02:57:06.381 INFO:teuthology.orchestra.run.smithi102.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-10T02:57:06.381 INFO:teuthology.orchestra.run.smithi102.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-10T02:57:06.381 INFO:teuthology.orchestra.run.smithi102.stdout:Creating mgr... 2024-04-10T02:57:06.381 INFO:teuthology.orchestra.run.smithi102.stdout:Verifying port 9283 ... 2024-04-10T02:57:06.553 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y 2024-04-10T02:57:06.553 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service: Unit ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service not loaded. 2024-04-10T02:57:06.701 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4.target.wants/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@.service. 2024-04-10T02:57:07.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/657052803' entity='client.admin' 2024-04-10T02:57:07.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:06 smithi102 podman[29522]: 2024-04-10 02:57:06.964944391 +0000 UTC m=+0.143878946 container create 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409) 2024-04-10T02:57:07.278 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-10T02:57:07.278 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout disabled 2024-04-10T02:57:07.285 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-10T02:57:07.286 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:57:07.286 INFO:teuthology.orchestra.run.smithi102.stdout:firewalld.service is not enabled 2024-04-10T02:57:07.286 INFO:teuthology.orchestra.run.smithi102.stdout:Not possible to enable service . firewalld.service is not available 2024-04-10T02:57:07.294 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-10T02:57:07.294 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout disabled 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:firewalld.service is not enabled 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mgr to start... 2024-04-10T02:57:07.302 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mgr... 2024-04-10T02:57:07.427 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 podman[29522]: 2024-04-10 02:57:07.167191679 +0000 UTC m=+0.346126188 container init 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-04-10T02:57:07.427 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 podman[29522]: 2024-04-10 02:57:07.169744929 +0000 UTC m=+0.348679444 container start 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-10T02:57:07.427 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 bash[29522]: 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 2024-04-10T02:57:07.427 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 systemd[1]: Started Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:57:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:07 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2230752316' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-10T02:57:08.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:07.682+0000 7f52ab5271c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T02:57:08.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:07.774+0000 7f52ab5271c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T02:57:08.438 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.415+0000 7f52ab5271c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T02:57:08.441 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:08.443 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsid": "e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4", 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "health": { 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 0 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "a" 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:08.444 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-10T02:57:08.445 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-10T02:57:08.449 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "restful" 2024-04-10T02:57:08.450 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modified": "2024-04-10T02:57:00.533232+0000", 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:08.451 INFO:teuthology.orchestra.run.smithi102.stdout:mgr not available, waiting (1/15)... 2024-04-10T02:57:08.732 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.504+0000 7f52ab5271c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T02:57:08.732 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.586+0000 7f52ab5271c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T02:57:09.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.733+0000 7f52ab5271c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T02:57:09.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.811+0000 7f52ab5271c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T02:57:09.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:08.978+0000 7f52ab5271c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T02:57:09.410 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:09 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:09.156+0000 7f52ab5271c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:09.410 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:09 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:09.237+0000 7f52ab5271c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T02:57:09.410 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:09 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:09.328+0000 7f52ab5271c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T02:57:09.410 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:09 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:09.410+0000 7f52ab5271c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T02:57:10.774 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:10.448+0000 7f52ab5271c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T02:57:10.775 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:10.586+0000 7f52ab5271c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T02:57:11.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/845659246' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-10T02:57:11.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:10.780+0000 7f52ab5271c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T02:57:11.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:11.018+0000 7f52ab5271c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T02:57:11.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:11.188+0000 7f52ab5271c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:11.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:11.366+0000 7f52ab5271c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T02:57:12.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:11.622+0000 7f52ab5271c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:12.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:11.705+0000 7f52ab5271c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:12.252 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:12.254 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:12.254 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsid": "e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4", 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "health": { 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-10T02:57:12.255 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 0 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "a" 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-10T02:57:12.261 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-10T02:57:12.262 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-10T02:57:12.263 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-10T02:57:12.264 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "restful" 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modified": "2024-04-10T02:57:00.533232+0000", 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:12.265 INFO:teuthology.orchestra.run.smithi102.stdout:mgr not available, waiting (2/15)... 2024-04-10T02:57:12.376 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:12.092+0000 7f52ab5271c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T02:57:13.166 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:12.872+0000 7f52ab5271c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T02:57:13.167 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:13.166+0000 7f52ab5271c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:13.481+0000 7f52ab5271c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:13.671+0000 7f52ab5271c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: Activating manager daemon y 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: mgrmap e2: y(active, starting, since 0.00298186s) 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:57:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: Manager daemon y is now available 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' 2024-04-10T02:57:13.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:13 smithi102 ceph-mon[29301]: from='mgr.14100 172.21.15.102:0/2195769748' entity='mgr.y' 2024-04-10T02:57:15.959 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:15 smithi102 ceph-mon[29301]: mgrmap e3: y(active, since 1.00696s) 2024-04-10T02:57:17.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:16 smithi102 ceph-mon[29301]: mgrmap e4: y(active, since 2s) 2024-04-10T02:57:18.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:18 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2662857293' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-10T02:57:21.468 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsid": "e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4", 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "health": { 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 0 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "a" 2024-04-10T02:57:21.469 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-10T02:57:21.470 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.471 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "restful" 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ], 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "modified": "2024-04-10T02:57:00.533232+0000", 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout }, 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:21.472 INFO:teuthology.orchestra.run.smithi102.stdout:mgr is available 2024-04-10T02:57:22.474 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/975807238' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-10T02:57:22.728 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [global] 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout fsid = e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout [osd] 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-10T02:57:22.729 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-10T02:57:22.730 INFO:teuthology.orchestra.run.smithi102.stdout:Enabling cephadm module... 2024-04-10T02:57:23.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:23 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1600407156' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-10T02:57:23.793 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setuser ceph since I am not root 2024-04-10T02:57:23.793 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setgroup ceph since I am not root 2024-04-10T02:57:24.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:23.942+0000 7f36da5831c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T02:57:24.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:24.032+0000 7f36da5831c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T02:57:24.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1600407156' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-10T02:57:24.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-mon[29301]: mgrmap e5: y(active, since 9s) 2024-04-10T02:57:24.735 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:24.659+0000 7f36da5831c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T02:57:24.993 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:24.748+0000 7f36da5831c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T02:57:24.993 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:24.834+0000 7f36da5831c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T02:57:24.993 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:24.993+0000 7f36da5831c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T02:57:25.246 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.074+0000 7f36da5831c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T02:57:25.246 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.247+0000 7f36da5831c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T02:57:25.511 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3793610410' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-10T02:57:25.511 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.429+0000 7f36da5831c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:25.511 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.512+0000 7f36da5831c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T02:57:25.708 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:25.708 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for the mgr to restart... 2024-04-10T02:57:25.709 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mgr epoch 5... 2024-04-10T02:57:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.604+0000 7f36da5831c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T02:57:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:25.691+0000 7f36da5831c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T02:57:27.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:26.742+0000 7f36da5831c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T02:57:27.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:26.871+0000 7f36da5831c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T02:57:27.455 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.062+0000 7f36da5831c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T02:57:27.455 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.285+0000 7f36da5831c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T02:57:27.455 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.456+0000 7f36da5831c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:27.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.634+0000 7f36da5831c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T02:57:28.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.890+0000 7f36da5831c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:28.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:27.974+0000 7f36da5831c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:28.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:28 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:28.380+0000 7f36da5831c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T02:57:29.428 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:29.142+0000 7f36da5831c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T02:57:29.428 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:29.428+0000 7f36da5831c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T02:57:30.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:29.737+0000 7f36da5831c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:29.927+0000 7f36da5831c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: Active manager daemon y restarted 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: Activating manager daemon y 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: osdmap e2: 0 total, 0 up, 0 in 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: mgrmap e6: y(active, starting, since 0.00310256s) 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T02:57:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: Manager daemon y is now available 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T02:57:30.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:29 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T02:57:31.244 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:30 smithi102 ceph-mon[29301]: Found migration_current of "None". Setting to last migration. 2024-04-10T02:57:31.244 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:30 smithi102 ceph-mon[29301]: mgrmap e7: y(active, since 1.00483s) 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:mgr epoch 5 is available 2024-04-10T02:57:31.548 INFO:teuthology.orchestra.run.smithi102.stdout:Setting orchestrator backend to cephadm... 2024-04-10T02:57:32.480 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-10T02:57:32.480 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:31] ENGINE Bus STARTING 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:31] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:31] ENGINE Bus STARTED 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:32.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:32 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:33.410 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:33 smithi102 ceph-mon[29301]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:33.410 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:33 smithi102 ceph-mon[29301]: mgrmap e8: y(active, since 2s) 2024-04-10T02:57:33.953 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-10T02:57:33.953 INFO:teuthology.orchestra.run.smithi102.stdout:Generating ssh key... 2024-04-10T02:57:34.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-mon[29301]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: Generating public/private rsa key pair. 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: Your identification has been saved in /tmp/tmpmom5lnpc/key. 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: Your public key has been saved in /tmp/tmpmom5lnpc/key.pub. 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: The key fingerprint is: 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: SHA256:j1amj049+p9YFW5O4VFNG4UADHvITkCTt2mQ2vDMc+s ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: The key's randomart image is: 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: +---[RSA 3072]----+ 2024-04-10T02:57:35.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | .+o.o.... +*| 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | . ++.o. ..+| 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | B o=o. +. | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | . *o=. o + | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | +S.o * | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | .B = | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | .= + . . | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | oE+ + . | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: | .+.+.o | 2024-04-10T02:57:35.044 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: +----[SHA256]-----+ 2024-04-10T02:57:35.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:35 smithi102 ceph-mon[29301]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:35.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:35 smithi102 ceph-mon[29301]: Generating ssh key... 2024-04-10T02:57:35.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:35 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:35.978 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:35 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:36.571 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtQJYVWVEci67Blgvqr2i8cp+UZgKK2Nw9aCCoi5gx4aR2Wq1Wyq8gHWq32/xMBNlSdDJEpJGQht0gs8WFwvIHyp/dxJWB+akryWGQg4g/r/DW4GXshJb3I9Fl6Ei0heO35C7XgK9spDh9HabXEOZijYgAAULxkJdW4t91txt30RAEQJwkvmsVT4x+lFTfp/ea6yRIlM8tX91P5LQggJtTmzT5YVqxn9LQIuHuHETpJM6vsJZLlSol9ej0vrIG1EPVLGVstVQQsyTa+GzVerwpGzCD24I/PUtLHot9ckxS9RLErbYzDOHHpoFAHPqX/93lmI2q5K7D0Ee/vrukwUfwVCLlpySmGL159lQm2+2I47zMXkZ//qv5a3S2bZg5tm84PrIN05e1ujE1W8sMWoP64Rkez0Smh5sFYVltdlkwXFaBiCsDmAoV0dnX25JYIYm8iEF7CebRdEH0LGBV+VHrkbG0Xh696CmKxJT+2ryaXuvnX81BAQ0Myf2yGNvscEc= ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:57:36.571 INFO:teuthology.orchestra.run.smithi102.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-10T02:57:36.571 INFO:teuthology.orchestra.run.smithi102.stdout:Adding key to root@localhost authorized_keys... 2024-04-10T02:57:36.571 INFO:teuthology.orchestra.run.smithi102.stdout:Adding host smithi102... 2024-04-10T02:57:37.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:36 smithi102 ceph-mon[29301]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:37.755 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:37 smithi102 ceph-mon[29301]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi102", "addr": "172.21.15.102", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:39.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:38 smithi102 ceph-mon[29301]: Deploying cephadm binary to smithi102 2024-04-10T02:57:39.841 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout Added host 'smithi102' with addr '172.21.15.102' 2024-04-10T02:57:39.841 INFO:teuthology.orchestra.run.smithi102.stdout:Deploying unmanaged mon service... 2024-04-10T02:57:40.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:40 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:40.453 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:40 smithi102 ceph-mon[29301]: Added host smithi102 2024-04-10T02:57:40.453 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:40 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:41.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:41 smithi102 ceph-mon[29301]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:41.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:41 smithi102 ceph-mon[29301]: Saving service mon spec with placement count:5 2024-04-10T02:57:41.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:41 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:42.116 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-10T02:57:42.116 INFO:teuthology.orchestra.run.smithi102.stdout:Deploying unmanaged mgr service... 2024-04-10T02:57:42.972 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:42 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:42.972 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:42 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:43.850 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-10T02:57:44.066 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:43 smithi102 ceph-mon[29301]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:44.066 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:43 smithi102 ceph-mon[29301]: Saving service mgr spec with placement count:2 2024-04-10T02:57:44.066 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:43 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:44.066 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:43 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:45.725 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:45 smithi102 ceph-mon[29301]: from='mgr.14118 172.21.15.102:0/1271108199' entity='mgr.y' 2024-04-10T02:57:45.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:45 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1869518411' entity='client.admin' 2024-04-10T02:57:46.900 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:46 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/104311229' entity='client.admin' 2024-04-10T02:57:47.178 INFO:teuthology.orchestra.run.smithi102.stdout:Enabling the dashboard module... 2024-04-10T02:57:48.207 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3876913246' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-10T02:57:48.207 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setuser ceph since I am not root 2024-04-10T02:57:48.207 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setgroup ceph since I am not root 2024-04-10T02:57:48.507 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:48.445+0000 7f866436e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T02:57:48.757 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:48.538+0000 7f866436e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T02:57:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:48 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3876913246' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-10T02:57:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:48 smithi102 ceph-mon[29301]: mgrmap e9: y(active, since 17s) 2024-04-10T02:57:49.509 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.180+0000 7f866436e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T02:57:49.509 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.274+0000 7f866436e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T02:57:49.509 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.357+0000 7f866436e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T02:57:49.509 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.509+0000 7f866436e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T02:57:49.762 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.591+0000 7f866436e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T02:57:49.763 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.763+0000 7f866436e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for the mgr to restart... 2024-04-10T02:57:49.767 INFO:teuthology.orchestra.run.smithi102.stdout:Waiting for mgr epoch 9... 2024-04-10T02:57:50.035 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1652507544' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-10T02:57:50.035 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:49 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:49.949+0000 7f866436e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:50.035 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:50.036+0000 7f866436e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T02:57:50.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:50.132+0000 7f866436e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T02:57:50.293 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:50.219+0000 7f866436e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T02:57:51.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:51.255+0000 7f866436e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T02:57:51.543 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:51.382+0000 7f866436e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T02:57:51.968 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:51.570+0000 7f866436e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T02:57:51.968 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:51.796+0000 7f866436e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T02:57:51.968 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:51.969+0000 7f866436e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:52.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:52.149+0000 7f866436e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T02:57:52.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:52.404+0000 7f866436e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:57:52.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:52.489+0000 7f866436e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T02:57:53.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:52.874+0000 7f866436e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T02:57:53.930 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:53.642+0000 7f866436e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T02:57:53.931 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:53.931+0000 7f866436e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T02:57:54.493 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:54.242+0000 7f866436e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T02:57:54.493 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T02:57:54.435+0000 7f866436e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: Active manager daemon y restarted 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: Activating manager daemon y 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: osdmap e3: 0 total, 0 up, 0 in 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: mgrmap e10: y(active, starting, since 0.00558801s) 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T02:57:54.494 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: Manager daemon y is now available 2024-04-10T02:57:54.495 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:54.495 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:57:54.495 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:54 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:54] ENGINE Bus STARTING 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:54] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: [10/Apr/2024:02:57:54] ENGINE Bus STARTED 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:55.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:55 smithi102 ceph-mon[29301]: mgrmap e11: y(active, since 1.00551s) 2024-04-10T02:57:56.068 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout { 2024-04-10T02:57:56.069 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-10T02:57:56.069 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-10T02:57:56.069 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout } 2024-04-10T02:57:56.069 INFO:teuthology.orchestra.run.smithi102.stdout:mgr epoch 9 is available 2024-04-10T02:57:56.069 INFO:teuthology.orchestra.run.smithi102.stdout:Generating a dashboard self-signed certificate... 2024-04-10T02:57:56.960 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:56 smithi102 ceph-mon[29301]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-10T02:57:56.960 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:56 smithi102 ceph-mon[29301]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-10T02:57:56.960 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:56 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:56.960 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:56 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:57.677 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-10T02:57:57.677 INFO:teuthology.orchestra.run.smithi102.stdout:Creating initial admin user... 2024-04-10T02:57:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:57 smithi102 ceph-mon[29301]: mgrmap e12: y(active, since 2s) 2024-04-10T02:57:58.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:57 smithi102 ceph-mon[29301]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:58.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:57 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:58.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:57 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:59.254 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:59 smithi102 ceph-mon[29301]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:57:59.254 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:57:59 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:57:59.529 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$c7nt.uRCjihKtSeNIoPOJ.N16RIvfgwZwegqtXo3SMoOsebq/COki", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712717878, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-10T02:57:59.529 INFO:teuthology.orchestra.run.smithi102.stdout:Fetching dashboard port number... 2024-04-10T02:58:00.482 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:00 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/18484210' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-10T02:58:00.770 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stdout 8443 2024-04-10T02:58:00.780 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-10T02:58:00.780 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout disabled 2024-04-10T02:58:00.787 INFO:teuthology.orchestra.run.smithi102.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-10T02:58:00.787 INFO:teuthology.orchestra.run.smithi102.stdout:systemctl: stdout inactive 2024-04-10T02:58:00.787 INFO:teuthology.orchestra.run.smithi102.stdout:firewalld.service is not enabled 2024-04-10T02:58:00.787 INFO:teuthology.orchestra.run.smithi102.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout:Ceph Dashboard is now available at: 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout: URL: https://smithi102.front.sepia.ceph.com:8443/ 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout: User: admin 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout: Password: zvx9s8ut9v 2024-04-10T02:58:00.789 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:00.790 INFO:teuthology.orchestra.run.smithi102.stdout:Saving cluster configuration to /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config directory 2024-04-10T02:58:00.790 INFO:teuthology.orchestra.run.smithi102.stdout:Enabling autotune for osd_memory_target 2024-04-10T02:58:02.275 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:02 smithi102 ceph-mon[29301]: mgrmap e13: y(active, since 6s) 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.267 INFO:teuthology.orchestra.run.smithi102.stdout:Or, if you are only running a single cluster on this host: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: ceph telemetry on 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout:For more information see: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:03.268 INFO:teuthology.orchestra.run.smithi102.stdout:Bootstrap complete. 2024-04-10T02:58:03.305 INFO:tasks.cephadm:Fetching config... 2024-04-10T02:58:03.305 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:03.305 DEBUG:teuthology.orchestra.run.smithi102:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-10T02:58:03.320 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-10T02:58:03.321 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:03.321 DEBUG:teuthology.orchestra.run.smithi102:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-10T02:58:03.375 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-10T02:58:03.375 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:03.375 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-10T02:58:03.445 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-10T02:58:03.445 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:03.445 DEBUG:teuthology.orchestra.run.smithi102:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-10T02:58:03.500 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-10T02:58:03.500 DEBUG:teuthology.orchestra.run.smithi102:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtQJYVWVEci67Blgvqr2i8cp+UZgKK2Nw9aCCoi5gx4aR2Wq1Wyq8gHWq32/xMBNlSdDJEpJGQht0gs8WFwvIHyp/dxJWB+akryWGQg4g/r/DW4GXshJb3I9Fl6Ei0heO35C7XgK9spDh9HabXEOZijYgAAULxkJdW4t91txt30RAEQJwkvmsVT4x+lFTfp/ea6yRIlM8tX91P5LQggJtTmzT5YVqxn9LQIuHuHETpJM6vsJZLlSol9ej0vrIG1EPVLGVstVQQsyTa+GzVerwpGzCD24I/PUtLHot9ckxS9RLErbYzDOHHpoFAHPqX/93lmI2q5K7D0Ee/vrukwUfwVCLlpySmGL159lQm2+2I47zMXkZ//qv5a3S2bZg5tm84PrIN05e1ujE1W8sMWoP64Rkez0Smh5sFYVltdlkwXFaBiCsDmAoV0dnX25JYIYm8iEF7CebRdEH0LGBV+VHrkbG0Xh696CmKxJT+2ryaXuvnX81BAQ0Myf2yGNvscEc= ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-10T02:58:03.578 INFO:teuthology.orchestra.run.smithi102.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtQJYVWVEci67Blgvqr2i8cp+UZgKK2Nw9aCCoi5gx4aR2Wq1Wyq8gHWq32/xMBNlSdDJEpJGQht0gs8WFwvIHyp/dxJWB+akryWGQg4g/r/DW4GXshJb3I9Fl6Ei0heO35C7XgK9spDh9HabXEOZijYgAAULxkJdW4t91txt30RAEQJwkvmsVT4x+lFTfp/ea6yRIlM8tX91P5LQggJtTmzT5YVqxn9LQIuHuHETpJM6vsJZLlSol9ej0vrIG1EPVLGVstVQQsyTa+GzVerwpGzCD24I/PUtLHot9ckxS9RLErbYzDOHHpoFAHPqX/93lmI2q5K7D0Ee/vrukwUfwVCLlpySmGL159lQm2+2I47zMXkZ//qv5a3S2bZg5tm84PrIN05e1ujE1W8sMWoP64Rkez0Smh5sFYVltdlkwXFaBiCsDmAoV0dnX25JYIYm8iEF7CebRdEH0LGBV+VHrkbG0Xh696CmKxJT+2ryaXuvnX81BAQ0Myf2yGNvscEc= ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:03.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtQJYVWVEci67Blgvqr2i8cp+UZgKK2Nw9aCCoi5gx4aR2Wq1Wyq8gHWq32/xMBNlSdDJEpJGQht0gs8WFwvIHyp/dxJWB+akryWGQg4g/r/DW4GXshJb3I9Fl6Ei0heO35C7XgK9spDh9HabXEOZijYgAAULxkJdW4t91txt30RAEQJwkvmsVT4x+lFTfp/ea6yRIlM8tX91P5LQggJtTmzT5YVqxn9LQIuHuHETpJM6vsJZLlSol9ej0vrIG1EPVLGVstVQQsyTa+GzVerwpGzCD24I/PUtLHot9ckxS9RLErbYzDOHHpoFAHPqX/93lmI2q5K7D0Ee/vrukwUfwVCLlpySmGL159lQm2+2I47zMXkZ//qv5a3S2bZg5tm84PrIN05e1ujE1W8sMWoP64Rkez0Smh5sFYVltdlkwXFaBiCsDmAoV0dnX25JYIYm8iEF7CebRdEH0LGBV+VHrkbG0Xh696CmKxJT+2ryaXuvnX81BAQ0Myf2yGNvscEc= ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-10T02:58:03.637 INFO:teuthology.orchestra.run.smithi152.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtQJYVWVEci67Blgvqr2i8cp+UZgKK2Nw9aCCoi5gx4aR2Wq1Wyq8gHWq32/xMBNlSdDJEpJGQht0gs8WFwvIHyp/dxJWB+akryWGQg4g/r/DW4GXshJb3I9Fl6Ei0heO35C7XgK9spDh9HabXEOZijYgAAULxkJdW4t91txt30RAEQJwkvmsVT4x+lFTfp/ea6yRIlM8tX91P5LQggJtTmzT5YVqxn9LQIuHuHETpJM6vsJZLlSol9ej0vrIG1EPVLGVstVQQsyTa+GzVerwpGzCD24I/PUtLHot9ckxS9RLErbYzDOHHpoFAHPqX/93lmI2q5K7D0Ee/vrukwUfwVCLlpySmGL159lQm2+2I47zMXkZ//qv5a3S2bZg5tm84PrIN05e1ujE1W8sMWoP64Rkez0Smh5sFYVltdlkwXFaBiCsDmAoV0dnX25JYIYm8iEF7CebRdEH0LGBV+VHrkbG0Xh696CmKxJT+2ryaXuvnX81BAQ0Myf2yGNvscEc= ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:03.652 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-10T02:58:03.841 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:03 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/312940693' entity='client.admin' 2024-04-10T02:58:03.908 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:06.658 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-10T02:58:06.659 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-10T02:58:07.061 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3500657553' entity='client.admin' 2024-04-10T02:58:07.128 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:09.891 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi152 2024-04-10T02:58:09.892 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:58:09.892 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.conf 2024-04-10T02:58:09.910 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:58:09.910 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:09.968 INFO:tasks.cephadm:Adding host smithi152 to orchestrator... 2024-04-10T02:58:09.968 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch host add smithi152 2024-04-10T02:58:10.227 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:10.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:10.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:10 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:11.266 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:11 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:13.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:13 smithi102 ceph-mon[29301]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi152", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:13 smithi102 ceph-mon[29301]: Deploying cephadm binary to smithi152 2024-04-10T02:58:14.141 INFO:teuthology.orchestra.run.smithi102.stdout:Added host 'smithi152' with addr '172.21.15.152' 2024-04-10T02:58:14.806 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-10T02:58:15.052 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: Added host smithi152 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:15.417 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:16.687 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T02:58:16.688 INFO:teuthology.orchestra.run.smithi102.stdout:[{"addr": "172.21.15.102", "hostname": "smithi102", "labels": [], "status": ""}, {"addr": "172.21.15.152", "hostname": "smithi152", "labels": [], "status": ""}] 2024-04-10T02:58:16.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:16 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:17.252 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-10T02:58:17.252 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-10T02:58:17.496 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:17.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:17 smithi102 ceph-mon[29301]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:17.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:17 smithi102 ceph-mon[29301]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T02:58:19.546 INFO:teuthology.orchestra.run.smithi102.stderr:adjusted tunables profile to default 2024-04-10T02:58:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1728420148' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-10T02:58:19.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:19.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:20.098 INFO:tasks.cephadm:Adding mon.a on smithi102 2024-04-10T02:58:20.098 INFO:tasks.cephadm:Adding mon.c on smithi102 2024-04-10T02:58:20.098 INFO:tasks.cephadm:Adding mon.b on smithi152 2024-04-10T02:58:20.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply mon '3;smithi102:172.21.15.102=a;smithi102:[v2:172.21.15.102:3301,v1:172.21.15.102:6790]=c;smithi152:172.21.15.152=b' 2024-04-10T02:58:20.345 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:21.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:20 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T02:58:21.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:20 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1728420148' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-10T02:58:21.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:20 smithi102 ceph-mon[29301]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:22.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:21 smithi102 ceph-mon[29301]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:22.372 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mon update... 2024-04-10T02:58:22.975 DEBUG:teuthology.orchestra.run.smithi102:mon.c> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.c.service 2024-04-10T02:58:22.978 DEBUG:teuthology.orchestra.run.smithi152:mon.b> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.b.service 2024-04-10T02:58:22.980 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-10T02:58:22.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mon dump -f json 2024-04-10T02:58:23.736 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi102:172.21.15.102=a;smithi102:[v2:172.21.15.102:3301,v1:172.21.15.102:6790]=c;smithi152:172.21.15.152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: Saving service mon spec with placement smithi102:172.21.15.102=a;smithi102:[v2:172.21.15.102:3301,v1:172.21.15.102:6790]=c;smithi152:172.21.15.152=b;count:3 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: Deploying daemon mon.b on smithi152 2024-04-10T02:58:23.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:23 smithi102 ceph-mon[29301]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:25 smithi102 ceph-mon[29301]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:27.006 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:58:27.006 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":1,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","modified":"2024-04-10T02:56:57.710085Z","created":"2024-04-10T02:56:57.710085Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:3300","nonce":0},{"type":"v1","addr":"172.21.15.102:6789","nonce":0}]},"addr":"172.21.15.102:6789/0","public_addr":"172.21.15.102:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-10T02:58:27.007 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 1 2024-04-10T02:58:27.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:27 smithi102 ceph-mon[29301]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:27.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:27 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/3992224991' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:27 smithi152 systemd[1]: Started Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:58:28.616 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-10T02:58:28.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mon dump -f json 2024-04-10T02:58:28.864 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: mon.a calling monitor election 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.198 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: mon.b calling monitor election 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: monmap e2: 2 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: fsmap 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: mgrmap e13: y(active, since 38s) 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: overall HEALTH_OK 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:33.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:32 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:33.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:33.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: mon.a calling monitor election 2024-04-10T02:58:33.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: mon.b calling monitor election 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: monmap e2: 2 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: fsmap 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: mgrmap e13: y(active, since 38s) 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: overall HEALTH_OK 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:33.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:33.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:32 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:33.535 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:58:33.536 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":2,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","modified":"2024-04-10T02:58:27.816631Z","created":"2024-04-10T02:56:57.710085Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:3300","nonce":0},{"type":"v1","addr":"172.21.15.102:6789","nonce":0}]},"addr":"172.21.15.102:6789/0","public_addr":"172.21.15.102:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-10T02:58:33.537 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 2 2024-04-10T02:58:34.121 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:33 smithi152 ceph-mon[33523]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:34.121 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:33 smithi152 ceph-mon[33523]: Deploying daemon mon.c on smithi102 2024-04-10T02:58:34.121 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:33 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/2635360007' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:34.121 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:33 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:34.255 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:33 smithi102 ceph-mon[29301]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:34.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:33 smithi102 ceph-mon[29301]: Deploying daemon mon.c on smithi102 2024-04-10T02:58:34.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:33 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/2635360007' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:34.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:33 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:35.150 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-10T02:58:35.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mon dump -f json 2024-04-10T02:58:35.410 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T02:58:35.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:35 smithi102 systemd[1]: Starting Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T02:58:36.037 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:35 smithi102 ceph-mon[29301]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.037 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:35 smithi102 podman[39632]: 2024-04-10 02:58:35.932716286 +0000 UTC m=+0.110820843 container create 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=quincy-d3b5060) 2024-04-10T02:58:36.038 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:35 smithi102 podman[39632]: 2024-04-10 02:58:35.839514973 +0000 UTC m=+0.017619530 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T02:58:36.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:35 smithi152 ceph-mon[33523]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 podman[39632]: 2024-04-10 02:58:36.127377274 +0000 UTC m=+0.305481841 container init 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 podman[39632]: 2024-04-10 02:58:36.130549435 +0000 UTC m=+0.308653993 container start 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: set uid:gid to 167:167 (ceph:ceph) 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable), process ceph-mon, pid 2 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pidfile_write: ignore empty --pid-file 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: load: jerasure load: lrc 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: RocksDB version: 6.15.5 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-10T02:58:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Compile date Apr 9 2024 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: DB SUMMARY 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: DB Session ID: MNHKVON3C5NYADAPHG6E 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: CURRENT file: CURRENT 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: IDENTITY file: IDENTITY 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.error_if_exists: 0 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.create_if_missing: 0 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.paranoid_checks: 1 2024-04-10T02:58:36.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.env: 0x55e730ffca80 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.fs: Posix File System 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.info_log: 0x55e7320d6e80 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_file_opening_threads: 16 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.statistics: (nil) 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.use_fsync: 0 2024-04-10T02:58:36.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_log_file_size: 0 2024-04-10T02:58:36.297 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-10T02:58:36.297 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-10T02:58:36.297 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.keep_log_file_num: 1000 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.recycle_log_file_num: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_fallocate: 1 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_mmap_reads: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_mmap_writes: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.use_direct_reads: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.create_missing_column_families: 0 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.db_log_dir: 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-10T02:58:36.298 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.advise_random_on_open: 1 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.db_write_buffer_size: 0 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.write_buffer_manager: 0x55e732e46930 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-10T02:58:36.299 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.rate_limiter: (nil) 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.wal_recovery_mode: 2 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.enable_thread_tracking: 0 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.enable_pipelined_write: 0 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.unordered_write: 0 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-10T02:58:36.300 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.row_cache: None 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.wal_filter: None 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_ingest_behind: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.preserve_deletes: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.two_write_queues: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.manual_wal_flush: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.atomic_flush: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-10T02:58:36.301 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.log_readahead_size: 0 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.best_efforts_recovery: 0 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.allow_data_in_errors: 0 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.db_host_id: __hostname__ 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_background_jobs: 2 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_background_compactions: -1 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_subcompactions: 1 2024-04-10T02:58:36.302 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_total_wal_size: 0 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_open_files: -1 2024-04-10T02:58:36.303 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bytes_per_sync: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_readahead_size: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_background_flushes: -1 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Compression algorithms supported: 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kZSTD supported: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kXpressCompression supported: 0 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kLZ4HCCompression supported: 1 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kZlibCompression supported: 1 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kSnappyCompression supported: 1 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kLZ4Compression supported: 1 2024-04-10T02:58:36.304 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: kBZip2Compression supported: 0 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.merge_operator: 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_filter: None 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_filter_factory: None 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.sst_partitioner_factory: None 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e732126f58) 2024-04-10T02:58:36.305 INFO:journalctl@ceph.mon.c.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_type: 0 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: data_block_index_type: 0 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_shortening: 1 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: hash_index_allow_collision: 1 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: checksum: 1 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: no_block_cache: 0 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache: 0x55e732159090 2024-04-10T02:58:36.306 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_options: 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: capacity : 536870912 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: num_shard_bits : 4 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: strict_capacity_limit : 0 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_compressed: (nil) 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: persistent_cache: (nil) 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_size: 4096 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_size_deviation: 10 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_restart_interval: 16 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_block_restart_interval: 1 2024-04-10T02:58:36.307 INFO:journalctl@ceph.mon.c.smithi102.stdout: metadata_block_size: 4096 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: partition_filters: 0 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: use_delta_encoding: 1 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: whole_key_filtering: 1 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: verify_compression: 0 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: format_version: 4 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: enable_index_compression: 1 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_align: 0 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.write_buffer_size: 33554432 2024-04-10T02:58:36.308 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_write_buffer_number: 2 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression: NoCompression 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression: Disabled 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.prefix_extractor: nullptr 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.num_levels: 7 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-10T02:58:36.309 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.level: 32767 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.strategy: 0 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compression_opts.enabled: false 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-10T02:58:36.310 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.target_file_size_base: 67108864 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-10T02:58:36.311 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.arena_block_size: 4194304 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.disable_auto_compactions: 0 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-10T02:58:36.312 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.table_properties_collectors: 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.inplace_update_support: 0 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-10T02:58:36.313 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.bloom_locality: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.max_successive_merges: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.paranoid_file_checks: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.force_consistency_checks: 1 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.report_bg_io_stats: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.ttl: 2592000 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.enable_blob_files: false 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.min_blob_size: 0 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.blob_file_size: 268435456 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-10T02:58:36.314 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717916157782, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717916158209, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712717916, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8e20b443-dae5-40e8-a2f8-6b427d9a5a70", "db_session_id": "MNHKVON3C5NYADAPHG6E"}} 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712717916158640, "job": 1, "event": "recovery_finished"} 2024-04-10T02:58:36.315 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55e7320ed180 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: DB pointer 0x55e7321c2000 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** DB Stats ** 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T02:58:36.316 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.317 INFO:journalctl@ceph.mon.c.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.318 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-10T02:58:36.319 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: using public_addrv [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] 2024-04-10T02:58:36.320 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: starting mon.c rank -1 at public addrs [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] at bind addrs [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(???) e0 preinit fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).mds e1 new map 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: e1 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: legacy client fscid: -1 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout: No filesystems configured 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-10T02:58:36.321 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-10T02:58:36.322 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-10T02:58:36.322 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-10T02:58:36.322 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-10T02:58:36.322 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-10T02:58:36.322 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1728420148' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi102:172.21.15.102=a;smithi102:[v2:172.21.15.102:3301,v1:172.21.15.102:6790]=c;smithi152:172.21.15.152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: Saving service mon spec with placement smithi102:172.21.15.102=a;smithi102:[v2:172.21.15.102:3301,v1:172.21.15.102:6790]=c;smithi152:172.21.15.152=b;count:3 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:36.323 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: Deploying daemon mon.b on smithi152 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/3992224991' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:36.324 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.a calling monitor election 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.b calling monitor election 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: monmap e2: 2 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: fsmap 2024-04-10T02:58:36.325 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mgrmap e13: y(active, since 38s) 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: overall HEALTH_OK 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: Deploying daemon mon.c on smithi102 2024-04-10T02:58:36.326 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/2635360007' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:36.327 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:36.327 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:36.327 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 ceph-mon[39651]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-10T02:58:36.327 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 bash[39632]: 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 2024-04-10T02:58:36.327 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:36 smithi102 systemd[1]: Started Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: mon.a calling monitor election 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: mon.b calling monitor election 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: mon.c calling monitor election 2024-04-10T02:58:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: monmap e3: 3 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: fsmap 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: mgrmap e13: y(active, since 46s) 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: overall HEALTH_OK 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:41 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: mon.a calling monitor election 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: mon.b calling monitor election 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: mon.c calling monitor election 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: monmap e3: 3 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: fsmap 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: mgrmap e13: y(active, since 46s) 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: overall HEALTH_OK 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T02:58:41.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: mon.a calling monitor election 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: mon.b calling monitor election 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: mon.c calling monitor election 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: monmap e3: 3 mons at {a=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-10T02:58:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: fsmap 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: osdmap e4: 0 total, 0 up, 0 in 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: mgrmap e13: y(active, since 46s) 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: overall HEALTH_OK 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:41 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:41.565 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-10T02:58:41.565 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":3,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","modified":"2024-04-10T02:58:36.178392Z","created":"2024-04-10T02:56:57.710085Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:3300","nonce":0},{"type":"v1","addr":"172.21.15.102:6789","nonce":0}]},"addr":"172.21.15.102:6789/0","public_addr":"172.21.15.102:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:3301","nonce":0},{"type":"v1","addr":"172.21.15.102:6790","nonce":0}]},"addr":"172.21.15.102:6790/0","public_addr":"172.21.15.102:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-10T02:58:41.568 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 3 2024-04-10T02:58:42.154 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-10T02:58:42.155 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-10T02:58:42.416 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:42 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:42 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/12587853' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:42 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/12587853' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/12587853' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-10T02:58:42.539 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:42 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T02:58:43.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.545 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:43.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:43.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:44.274 INFO:teuthology.orchestra.run.smithi102.stdout:# minimal ceph.conf for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:44.274 INFO:teuthology.orchestra.run.smithi102.stdout:[global] 2024-04-10T02:58:44.274 INFO:teuthology.orchestra.run.smithi102.stdout: fsid = e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T02:58:44.274 INFO:teuthology.orchestra.run.smithi102.stdout: mon_host = [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] 2024-04-10T02:58:44.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: Reconfiguring mon.a (unknown last config time)... 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: Reconfiguring daemon mon.a on smithi102 2024-04-10T02:58:44.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:44 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/320046085' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: Reconfiguring mon.a (unknown last config time)... 2024-04-10T02:58:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: Reconfiguring daemon mon.a on smithi102 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/320046085' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: Reconfiguring mon.a (unknown last config time)... 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: Reconfiguring daemon mon.a on smithi102 2024-04-10T02:58:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:44 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/320046085' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:45.306 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-10T02:58:45.307 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:45.307 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.conf 2024-04-10T02:58:45.337 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:45.338 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:45.405 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:58:45.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.conf 2024-04-10T02:58:45.439 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:58:45.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-10T02:58:45.510 INFO:tasks.cephadm:Adding mgr.y on smithi102 2024-04-10T02:58:45.511 INFO:tasks.cephadm:Adding mgr.x on smithi152 2024-04-10T02:58:45.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply mgr '2;smithi102=y;smithi152=x' 2024-04-10T02:58:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:45 smithi102 ceph-mon[39651]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:45.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:45 smithi102 ceph-mon[29301]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:45.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:45 smithi152 ceph-mon[33523]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:45.797 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: Reconfiguring mon.c (monmap changed)... 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:46 smithi152 ceph-mon[33523]: Reconfiguring daemon mon.c on smithi102 2024-04-10T02:58:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: Reconfiguring mon.c (monmap changed)... 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[29301]: Reconfiguring daemon mon.c on smithi102 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: Reconfiguring mon.c (monmap changed)... 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:47.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:47.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:46 smithi102 ceph-mon[39651]: Reconfiguring daemon mon.c on smithi102 2024-04-10T02:58:47.397 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mgr update... 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:47.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.001 DEBUG:teuthology.orchestra.run.smithi152:mgr.x> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service 2024-04-10T02:58:48.003 INFO:tasks.cephadm:Deploying OSDs... 2024-04-10T02:58:48.003 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T02:58:48.003 DEBUG:teuthology.orchestra.run.smithi102:> dd if=/scratch_devs of=/dev/stdout 2024-04-10T02:58:48.022 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-10T02:58:48.022 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_1 2024-04-10T02:58:48.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:48.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T02:58:48.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T02:58:48.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:48.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-04-10 02:58:35.661202656 +0000 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-04-10 02:53:13.605126866 +0000 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-04-10 02:53:13.605126866 +0000 2024-04-10T02:58:48.060 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-04-10 02:53:13.605126866 +0000 2024-04-10T02:58:48.061 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-10T02:58:48.129 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-04-10T02:58:48.129 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-04-10T02:58:48.129 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000201001 s, 2.5 MB/s 2024-04-10T02:58:48.130 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-10T02:58:48.187 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_2 2024-04-10T02:58:48.243 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-10T02:58:48.243 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.243 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-04-10 02:58:35.661202656 +0000 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-04-10 02:53:13.951119182 +0000 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-04-10 02:53:13.951119182 +0000 2024-04-10T02:58:48.244 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-04-10 02:53:13.951119182 +0000 2024-04-10T02:58:48.244 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-10T02:58:48.312 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-04-10T02:58:48.312 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-04-10T02:58:48.312 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000174827 s, 2.9 MB/s 2024-04-10T02:58:48.312 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-10T02:58:48.368 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_3 2024-04-10T02:58:48.423 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-04-10 02:58:35.662202635 +0000 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-04-10 02:53:14.281111852 +0000 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-04-10 02:53:14.281111852 +0000 2024-04-10T02:58:48.424 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-04-10 02:53:14.281111852 +0000 2024-04-10T02:58:48.424 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-10T02:58:48.491 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-04-10T02:58:48.491 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-04-10T02:58:48.491 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000205615 s, 2.5 MB/s 2024-04-10T02:58:48.492 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-10T02:58:48.549 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_4 2024-04-10T02:58:48.603 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-04-10 02:58:35.662202635 +0000 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-04-10 02:53:14.611104522 +0000 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-04-10 02:53:14.611104522 +0000 2024-04-10T02:58:48.604 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-04-10 02:53:14.611104522 +0000 2024-04-10T02:58:48.605 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-10T02:58:48.669 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-04-10T02:58:48.669 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-04-10T02:58:48.669 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000222624 s, 2.3 MB/s 2024-04-10T02:58:48.670 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-10T02:58:48.726 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T02:58:48.726 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/scratch_devs of=/dev/stdout 2024-04-10T02:58:48.742 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-10T02:58:48.743 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_1 2024-04-10T02:58:48.798 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-10T02:58:48.798 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.798 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-10T02:58:48.798 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.798 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.799 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-10 02:58:26.447788596 +0000 2024-04-10T02:58:48.799 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-10 02:53:22.906571532 +0000 2024-04-10T02:58:48.799 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-10 02:53:22.906571532 +0000 2024-04-10T02:58:48.799 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-10 02:53:22.906571532 +0000 2024-04-10T02:58:48.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-10T02:58:48.866 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-10T02:58:48.867 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-10T02:58:48.867 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000252766 s, 2.0 MB/s 2024-04-10T02:58:48.868 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-10T02:58:48.925 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_2 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-10 02:58:26.447788596 +0000 2024-04-10T02:58:48.981 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-10 02:53:23.248566738 +0000 2024-04-10T02:58:48.982 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-10 02:53:23.248566738 +0000 2024-04-10T02:58:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-10 02:53:23.248566738 +0000 2024-04-10T02:58:48.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-10T02:58:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[39651]: Reconfiguring mon.b (monmap changed)... 2024-04-10T02:58:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[39651]: Reconfiguring daemon mon.b on smithi152 2024-04-10T02:58:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[39651]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi102=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[39651]: Saving service mgr spec with placement smithi102=y;smithi152=x;count:2 2024-04-10T02:58:49.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[29301]: Reconfiguring mon.b (monmap changed)... 2024-04-10T02:58:49.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[29301]: Reconfiguring daemon mon.b on smithi152 2024-04-10T02:58:49.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[29301]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi102=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:49.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:48 smithi102 ceph-mon[29301]: Saving service mgr spec with placement smithi102=y;smithi152=x;count:2 2024-04-10T02:58:49.045 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:48 smithi152 ceph-mon[33523]: Reconfiguring mon.b (monmap changed)... 2024-04-10T02:58:49.045 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:48 smithi152 ceph-mon[33523]: Reconfiguring daemon mon.b on smithi152 2024-04-10T02:58:49.045 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:48 smithi152 ceph-mon[33523]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi102=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:49.045 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:48 smithi152 ceph-mon[33523]: Saving service mgr spec with placement smithi102=y;smithi152=x;count:2 2024-04-10T02:58:49.047 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-10T02:58:49.048 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-10T02:58:49.048 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000164484 s, 3.1 MB/s 2024-04-10T02:58:49.049 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-10T02:58:49.104 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_3 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-10 02:58:26.447788596 +0000 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-10 02:53:23.597561846 +0000 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-10 02:53:23.597561846 +0000 2024-04-10T02:58:49.160 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-10 02:53:23.597561846 +0000 2024-04-10T02:58:49.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-10T02:58:49.226 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-10T02:58:49.227 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-10T02:58:49.227 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000240822 s, 2.1 MB/s 2024-04-10T02:58:49.228 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-10T02:58:49.285 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_4 2024-04-10T02:58:49.341 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-10T02:58:49.341 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-10 02:58:26.448788583 +0000 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-10 02:53:23.929557192 +0000 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-10 02:53:23.929557192 +0000 2024-04-10T02:58:49.342 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-10 02:53:23.929557192 +0000 2024-04-10T02:58:49.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-10T02:58:49.408 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-10T02:58:49.408 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-10T02:58:49.408 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000187253 s, 2.7 MB/s 2024-04-10T02:58:49.409 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-10T02:58:49.465 INFO:tasks.cephadm:Deploying osd.0 on smithi102 with /dev/vg_nvme/lv_4... 2024-04-10T02:58:49.465 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-10T02:58:49.718 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:50.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:50.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:50.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:50.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:50 smithi152 systemd[1]: Starting Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T02:58:51.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:50 smithi102 ceph-mon[29301]: Deploying daemon mgr.x on smithi152 2024-04-10T02:58:51.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:50 smithi102 ceph-mon[39651]: Deploying daemon mgr.x on smithi152 2024-04-10T02:58:51.080 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:50 smithi152 ceph-mon[33523]: Deploying daemon mgr.x on smithi152 2024-04-10T02:58:51.080 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:50 smithi152 podman[35367]: 2024-04-10 02:58:50.792977481 +0000 UTC m=+0.017675774 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T02:58:51.080 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:50 smithi152 podman[35367]: 2024-04-10 02:58:50.895032576 +0000 UTC m=+0.119730854 container create a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T02:58:51.080 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 podman[35367]: 2024-04-10 02:58:51.081805108 +0000 UTC m=+0.306503392 container init a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-10T02:58:51.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 podman[35367]: 2024-04-10 02:58:51.084780801 +0000 UTC m=+0.309479085 container start a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-04-10T02:58:51.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mgr[35386]: -- 172.21.15.152:0/1317990402 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5616e1a529c0 con 0x5616e1a28800 2024-04-10T02:58:51.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 bash[35367]: a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 2024-04-10T02:58:51.513 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 systemd[1]: Started Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T02:58:51.580 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 10+0 records in 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr:10+0 records out 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125485 s, 836 MB/s 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 2024-04-10T02:58:51.581 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping successful for: 2024-04-10T02:58:51.946 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:51 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:51.606+0000 7f514482a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:51.712+0000 7f514482a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:51.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:51 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:52.274 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi102:vg_nvme/lv_4 2024-04-10T02:58:52.527 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:58:52.703 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.370+0000 7f514482a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T02:58:52.703 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.467+0000 7f514482a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T02:58:52.703 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.551+0000 7f514482a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T02:58:53.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.703+0000 7f514482a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T02:58:53.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.783+0000 7f514482a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T02:58:53.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:52.951+0000 7f514482a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T02:58:53.389 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:53.131+0000 7f514482a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T02:58:53.389 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:53.214+0000 7f514482a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T02:58:53.390 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:53.305+0000 7f514482a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T02:58:53.390 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:53.390+0000 7f514482a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T02:58:53.638 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:53.638 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.638 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.638 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: Reconfiguring mgr.y (unknown last config time)... 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[39651]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.639 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: Reconfiguring mgr.y (unknown last config time)... 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.640 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:53 smithi102 ceph-mon[29301]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T02:58:53.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:53.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: Reconfiguring mgr.y (unknown last config time)... 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:53.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:53 smithi152 ceph-mon[33523]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T02:58:54.754 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:54 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:54.438+0000 7f514482a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T02:58:54.754 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:54 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:54.566+0000 7f514482a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T02:58:54.755 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:54 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:54.755+0000 7f514482a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T02:58:55.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:54 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:54.986+0000 7f514482a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T02:58:55.488 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:55.157+0000 7f514482a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:58:55.488 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:55.334+0000 7f514482a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T02:58:55.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:55.589+0000 7f514482a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T02:58:55.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:55.672+0000 7f514482a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:58:55.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:55 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:58:55.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:58:55.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:58:55.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:58:55.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:55 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:58:56.492 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:56 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:56.060+0000 7f514482a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T02:58:56.679 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:56 smithi102 ceph-mon[29301]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:56.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:56 smithi102 ceph-mon[39651]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:56.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:56 smithi152 ceph-mon[33523]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:58:57.119 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:56 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:56.832+0000 7f514482a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T02:58:57.119 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:57 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:57.120+0000 7f514482a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T02:58:57.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:57 smithi152 ceph-mon[33523]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:57.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:57 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:57.433+0000 7f514482a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T02:58:57.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 02:58:57 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T02:58:57.622+0000 7f514482a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T02:58:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:57 smithi102 ceph-mon[29301]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:57.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:57 smithi102 ceph-mon[39651]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]': finished 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: osdmap e5: 1 total, 0 up, 1 in 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:58:58.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:58 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2027604156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:58:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: Standby manager daemon x started 2024-04-10T02:58:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T02:58:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T02:58:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]': finished 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: osdmap e5: 1 total, 0 up, 1 in 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2027604156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1711798179' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]: dispatch 2024-04-10T02:58:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3093218352' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189ba4fa-2bb5-4bd0-a97b-bc549c5a9311"}]': finished 2024-04-10T02:58:58.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: osdmap e5: 1 total, 0 up, 1 in 2024-04-10T02:58:58.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:58:58.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:58 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2027604156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:58:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:59 smithi152 ceph-mon[33523]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:59.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:59 smithi152 ceph-mon[33523]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-10T02:58:59.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:58:59 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T02:58:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[29301]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:59.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[29301]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-10T02:58:59.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T02:58:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[39651]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:58:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[39651]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-10T02:58:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:58:59 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T02:59:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:01 smithi152 ceph-mon[33523]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:01.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:01 smithi102 ceph-mon[29301]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:01.787 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:01 smithi102 ceph-mon[39651]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:03 smithi152 ceph-mon[33523]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:03 smithi102 ceph-mon[29301]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:03 smithi102 ceph-mon[39651]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:04.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T02:59:04.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:04.586 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T02:59:04.586 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:04.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T02:59:04.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:05.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:05 smithi152 ceph-mon[33523]: Deploying daemon osd.0 on smithi102 2024-04-10T02:59:05.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:05 smithi152 ceph-mon[33523]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:05 smithi102 ceph-mon[29301]: Deploying daemon osd.0 on smithi102 2024-04-10T02:59:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:05 smithi102 ceph-mon[29301]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:05 smithi102 ceph-mon[39651]: Deploying daemon osd.0 on smithi102 2024-04-10T02:59:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:05 smithi102 ceph-mon[39651]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:07.728 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:07 smithi102 ceph-mon[29301]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:07.728 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:07 smithi102 ceph-mon[39651]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:07 smithi152 ceph-mon[33523]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:09.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[39651]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[29301]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:09.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:09 smithi152 ceph-mon[33523]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:11.677 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:11 smithi102 ceph-mon[29301]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:11.677 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:11 smithi102 ceph-mon[39651]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:11 smithi152 ceph-mon[33523]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:12.431 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 0 on host 'smithi102' 2024-04-10T02:59:13.090 DEBUG:teuthology.orchestra.run.smithi102:osd.0> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.0.service 2024-04-10T02:59:13.092 INFO:tasks.cephadm:Deploying osd.1 on smithi102 with /dev/vg_nvme/lv_3... 2024-04-10T02:59:13.092 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-10T02:59:13.397 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.671 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.671 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T02:59:13.671 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:13.671 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.671 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T02:59:13.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:13.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:13.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:14.715 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[29301]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[29301]: osdmap e6: 1 total, 0 up, 1 in 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[29301]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[39651]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[39651]: osdmap e6: 1 total, 0 up, 1 in 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:14.716 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-mon[39651]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:14.716 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[43548]: 2024-04-10T02:59:14.439+0000 7fb70fbfc700 -1 osd.0 0 waiting for initial osdmap 2024-04-10T02:59:14.717 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 02:59:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[43548]: 2024-04-10T02:59:14.446+0000 7fb7091f4700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T02:59:14.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:14 smithi152 ceph-mon[33523]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T02:59:14.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:14 smithi152 ceph-mon[33523]: osdmap e6: 1 total, 0 up, 1 in 2024-04-10T02:59:14.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:14 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:14.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:14 smithi152 ceph-mon[33523]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 10+0 records in 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr:10+0 records out 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0152089 s, 689 MB/s 2024-04-10T02:59:15.281 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping successful for: 2024-04-10T02:59:15.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:15 smithi152 ceph-mon[33523]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:15.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:15 smithi152 ceph-mon[33523]: osdmap e7: 1 total, 0 up, 1 in 2024-04-10T02:59:15.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:15 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:15 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:15 smithi152 ceph-mon[33523]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[29301]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[29301]: osdmap e7: 1 total, 0 up, 1 in 2024-04-10T02:59:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[29301]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[39651]: from='osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[39651]: osdmap e7: 1 total, 0 up, 1 in 2024-04-10T02:59:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:15 smithi102 ceph-mon[39651]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-10T02:59:15.889 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi102:vg_nvme/lv_3 2024-04-10T02:59:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T02:59:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319] boot 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: osdmap e8: 1 total, 1 up, 1 in 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319] boot 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: osdmap e8: 1 total, 1 up, 1 in 2024-04-10T02:59:16.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:16 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T02:59:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T02:59:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: osd.0 [v2:172.21.15.102:6802/3631766319,v1:172.21.15.102:6803/3631766319] boot 2024-04-10T02:59:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: osdmap e8: 1 total, 1 up, 1 in 2024-04-10T02:59:16.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:16 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T02:59:17.108 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:59:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:17 smithi102 ceph-mon[29301]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:17 smithi102 ceph-mon[29301]: osdmap e9: 1 total, 1 up, 1 in 2024-04-10T02:59:17.482 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:17 smithi102 ceph-mon[39651]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:17.482 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:17 smithi102 ceph-mon[39651]: osdmap e9: 1 total, 1 up, 1 in 2024-04-10T02:59:17.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:17 smithi152 ceph-mon[33523]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:17.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:17 smithi152 ceph-mon[33523]: osdmap e9: 1 total, 1 up, 1 in 2024-04-10T02:59:19.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi102 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi102 to 16064M 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi102 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi102 to 16064M 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi102 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi102 to 16064M 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:20.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[29301]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:20.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:20.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:20.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:20.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[39651]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:20.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:20.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:20.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:20.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:20 smithi152 ceph-mon[33523]: from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:20.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:20.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:20.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:21 smithi102 ceph-mon[29301]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:21.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:21 smithi102 ceph-mon[39651]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:21.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:21 smithi152 ceph-mon[33523]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]: dispatch 2024-04-10T02:59:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]': finished 2024-04-10T02:59:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:22 smithi152 ceph-mon[33523]: osdmap e10: 2 total, 1 up, 2 in 2024-04-10T02:59:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:22 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:22.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]: dispatch 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]': finished 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[39651]: osdmap e10: 2 total, 1 up, 2 in 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]: dispatch 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1558086169' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b295353c-a779-4ced-917a-64e180c8b379"}]': finished 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[29301]: osdmap e10: 2 total, 1 up, 2 in 2024-04-10T02:59:22.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:22 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:23.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:23 smithi152 ceph-mon[33523]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:23.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:23 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/327241511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:23.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:23 smithi102 ceph-mon[29301]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:23.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:23 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/327241511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:23.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:23 smithi102 ceph-mon[39651]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:23.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:23 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/327241511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:25.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:25 smithi152 ceph-mon[33523]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:25 smithi102 ceph-mon[39651]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:25.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:25 smithi102 ceph-mon[29301]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:27 smithi152 ceph-mon[33523]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:27.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:27 smithi102 ceph-mon[29301]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:27.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:27 smithi102 ceph-mon[39651]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:28.787 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T02:59:28.787 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:28.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T02:59:28.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T02:59:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:29.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:29 smithi102 ceph-mon[29301]: Deploying daemon osd.1 on smithi102 2024-04-10T02:59:29.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:29 smithi102 ceph-mon[29301]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:29.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:29 smithi102 ceph-mon[39651]: Deploying daemon osd.1 on smithi102 2024-04-10T02:59:29.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:29 smithi102 ceph-mon[39651]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:30.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:29 smithi152 ceph-mon[33523]: Deploying daemon osd.1 on smithi102 2024-04-10T02:59:30.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:29 smithi152 ceph-mon[33523]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:31.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:31 smithi102 ceph-mon[29301]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:31.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:31 smithi102 ceph-mon[39651]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:32.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:31 smithi152 ceph-mon[33523]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[29301]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[39651]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:33 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:33.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:33 smithi152 ceph-mon[33523]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:33.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:33 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:33 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:33.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:33 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:35.716 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:35 smithi102 ceph-mon[29301]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:35.716 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:35 smithi102 ceph-mon[39651]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:36.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:35 smithi152 ceph-mon[33523]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:36.453 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 1 on host 'smithi102' 2024-04-10T02:59:36.764 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[29301]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:36.764 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[29301]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:36.764 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:36.764 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:36.765 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[39651]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:36.765 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[39651]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:36.765 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:36.765 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:36 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:36 smithi152 ceph-mon[33523]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:37.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:36 smithi152 ceph-mon[33523]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T02:59:37.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:36 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:36 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.106 DEBUG:teuthology.orchestra.run.smithi102:osd.1> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.1.service 2024-04-10T02:59:37.108 INFO:tasks.cephadm:Deploying osd.2 on smithi102 with /dev/vg_nvme/lv_2... 2024-04-10T02:59:37.108 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-10T02:59:37.409 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:59:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: osdmap e11: 2 total, 1 up, 2 in 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: osdmap e11: 2 total, 1 up, 2 in 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:37.794 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[49661]: 2024-04-10T02:59:37.538+0000 7f5b8d125700 -1 osd.1 0 waiting for initial osdmap 2024-04-10T02:59:37.794 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 02:59:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[49661]: 2024-04-10T02:59:37.540+0000 7f5b89723700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T02:59:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T02:59:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: osdmap e11: 2 total, 1 up, 2 in 2024-04-10T02:59:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:38.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T02:59:38.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:38.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:37 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[29301]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[29301]: osdmap e12: 2 total, 1 up, 2 in 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[39651]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[39651]: osdmap e12: 2 total, 1 up, 2 in 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:38.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:38 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:38 smithi152 ceph-mon[33523]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T02:59:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:38 smithi152 ceph-mon[33523]: osdmap e12: 2 total, 1 up, 2 in 2024-04-10T02:59:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:38 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:38 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 10+0 records in 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr:10+0 records out 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123926 s, 846 MB/s 2024-04-10T02:59:39.328 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping successful for: 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406] boot 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: osdmap e13: 2 total, 2 up, 2 in 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:39.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406] boot 2024-04-10T02:59:39.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: osdmap e13: 2 total, 2 up, 2 in 2024-04-10T02:59:39.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:39 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T02:59:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T02:59:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-10T02:59:40.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: osd.1 [v2:172.21.15.102:6810/2661864406,v1:172.21.15.102:6811/2661864406] boot 2024-04-10T02:59:40.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: osdmap e13: 2 total, 2 up, 2 in 2024-04-10T02:59:40.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:39 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T02:59:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:40 smithi102 ceph-mon[39651]: osdmap e14: 2 total, 2 up, 2 in 2024-04-10T02:59:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:40 smithi102 ceph-mon[29301]: osdmap e14: 2 total, 2 up, 2 in 2024-04-10T02:59:40.866 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi102:vg_nvme/lv_2 2024-04-10T02:59:41.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:40 smithi152 ceph-mon[33523]: osdmap e14: 2 total, 2 up, 2 in 2024-04-10T02:59:41.309 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T02:59:41.688 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:41 smithi102 ceph-mon[29301]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:41.688 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:41 smithi102 ceph-mon[39651]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:41 smithi152 ceph-mon[33523]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T02:59:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi102 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi102 to 8032M 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi102 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi102 to 8032M 2024-04-10T02:59:44.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:44.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:44.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:44.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:44 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi102 2024-04-10T02:59:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi102 to 8032M 2024-04-10T02:59:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T02:59:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T02:59:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T02:59:45.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:44 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:45.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:45 smithi102 ceph-mon[29301]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:45.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:45 smithi102 ceph-mon[39651]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:45 smithi152 ceph-mon[33523]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:47.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:46 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2039586695' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:46 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:46 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]': finished 2024-04-10T02:59:47.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:46 smithi152 ceph-mon[33523]: osdmap e15: 3 total, 2 up, 3 in 2024-04-10T02:59:47.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:46 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T02:59:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2039586695' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]': finished 2024-04-10T02:59:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[29301]: osdmap e15: 3 total, 2 up, 3 in 2024-04-10T02:59:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T02:59:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2039586695' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]: dispatch 2024-04-10T02:59:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3601bd6b-a4e3-41ba-8688-15dcc451f5e8"}]': finished 2024-04-10T02:59:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[39651]: osdmap e15: 3 total, 2 up, 3 in 2024-04-10T02:59:47.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:46 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T02:59:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:47 smithi152 ceph-mon[33523]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:47 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3835302989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:48.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:47 smithi102 ceph-mon[39651]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:48.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:47 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3835302989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:47 smithi102 ceph-mon[29301]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:48.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3835302989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T02:59:49.679 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:49 smithi102 ceph-mon[29301]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:49.680 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:49 smithi102 ceph-mon[39651]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:49 smithi152 ceph-mon[33523]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:51.790 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:51 smithi102 ceph-mon[29301]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:51.790 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:51 smithi102 ceph-mon[39651]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:52.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:51 smithi152 ceph-mon[33523]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:52 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T02:59:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:52 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:52.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:52 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T02:59:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:52 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:52 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T02:59:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:52 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T02:59:53.897 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:53 smithi102 ceph-mon[29301]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:53.898 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:53 smithi102 ceph-mon[29301]: Deploying daemon osd.2 on smithi102 2024-04-10T02:59:53.898 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:53 smithi102 ceph-mon[39651]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:53.898 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:53 smithi102 ceph-mon[39651]: Deploying daemon osd.2 on smithi102 2024-04-10T02:59:54.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:53 smithi152 ceph-mon[33523]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:54.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:53 smithi152 ceph-mon[33523]: Deploying daemon osd.2 on smithi102 2024-04-10T02:59:55.885 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:55 smithi102 ceph-mon[39651]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:55.885 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:55 smithi102 ceph-mon[29301]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:55 smithi152 ceph-mon[33523]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[29301]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:57.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[39651]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:57.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:57.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:57 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:57 smithi152 ceph-mon[33523]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:57 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:57 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T02:59:58.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:58 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:58.935 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:58 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:59.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:58 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T02:59:59.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 02:59:59 smithi102 ceph-mon[29301]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T02:59:59.948 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 02:59:59 smithi102 ceph-mon[39651]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:00.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 02:59:59 smithi152 ceph-mon[33523]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:00.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: overall HEALTH_OK 2024-04-10T03:00:00.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: overall HEALTH_OK 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:00.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:00 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: overall HEALTH_OK 2024-04-10T03:00:01.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:01.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:01.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:00 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.156 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 2 on host 'smithi102' 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[29301]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[29301]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[39651]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[39651]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:01.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:01.815 DEBUG:teuthology.orchestra.run.smithi102:osd.2> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.2.service 2024-04-10T03:00:01.818 INFO:tasks.cephadm:Deploying osd.3 on smithi102 with /dev/vg_nvme/lv_1... 2024-04-10T03:00:01.818 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-10T03:00:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:01 smithi152 ceph-mon[33523]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:01 smithi152 ceph-mon[33523]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:00:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:02.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[39651]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:00:02.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[39651]: osdmap e16: 3 total, 2 up, 3 in 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[39651]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[29301]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[29301]: osdmap e16: 3 total, 2 up, 3 in 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:02.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-mon[29301]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:02.793 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[55292]: 2024-04-10T03:00:02.622+0000 7fb76fa0e700 -1 osd.2 0 waiting for initial osdmap 2024-04-10T03:00:02.793 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[55292]: 2024-04-10T03:00:02.625+0000 7fb769807700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:00:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:02 smithi152 ceph-mon[33523]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:00:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:02 smithi152 ceph-mon[33523]: osdmap e16: 3 total, 2 up, 3 in 2024-04-10T03:00:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:02 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:02 smithi152 ceph-mon[33523]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:03.515 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:00:03.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:03 smithi152 ceph-mon[33523]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:03.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:03 smithi152 ceph-mon[33523]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:03.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:03 smithi152 ceph-mon[33523]: osdmap e17: 3 total, 2 up, 3 in 2024-04-10T03:00:03.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:03 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:03 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[29301]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[29301]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[29301]: osdmap e17: 3 total, 2 up, 3 in 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[39651]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[39651]: from='osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[39651]: osdmap e17: 3 total, 2 up, 3 in 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:03.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:03 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:04 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:00:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:04 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:00:05.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:04 smithi152 ceph-mon[33523]: osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761] boot 2024-04-10T03:00:05.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:04 smithi152 ceph-mon[33523]: osdmap e18: 3 total, 3 up, 3 in 2024-04-10T03:00:05.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:00:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:00:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[29301]: osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761] boot 2024-04-10T03:00:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[29301]: osdmap e18: 3 total, 3 up, 3 in 2024-04-10T03:00:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:05.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:00:05.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:00:05.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[39651]: osd.2 [v2:172.21.15.102:6818/2522010761,v1:172.21.15.102:6819/2522010761] boot 2024-04-10T03:00:05.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[39651]: osdmap e18: 3 total, 3 up, 3 in 2024-04-10T03:00:05.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:00:06.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: pgmap v73: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: osdmap e19: 3 total, 3 up, 3 in 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi102 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi102 to 5354M 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: pgmap v73: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: osdmap e19: 3 total, 3 up, 3 in 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi102 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi102 to 5354M 2024-04-10T03:00:06.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:06.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:06.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:05 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr: stderr: 10+0 records in 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr:10+0 records out 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112359 s, 933 MB/s 2024-04-10T03:00:06.104 INFO:teuthology.orchestra.run.smithi102.stderr:--> Zapping successful for: 2024-04-10T03:00:06.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: pgmap v73: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: osdmap e19: 3 total, 3 up, 3 in 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi102 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi102 to 5354M 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:06.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:06.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:05 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:06.454 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59092]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:06.454 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59092]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:06.454 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59092]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:06.454 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59092]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:06.747 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59097]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:06.747 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59097]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:06.747 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59097]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:06.747 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59097]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:06.773 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi102:vg_nvme/lv_1 2024-04-10T03:00:07.023 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:00:07.023 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[29301]: osdmap e20: 3 total, 3 up, 3 in 2024-04-10T03:00:07.023 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:07.023 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:00:07.024 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[39651]: osdmap e20: 3 total, 3 up, 3 in 2024-04-10T03:00:07.024 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:07.024 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59101]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:07.024 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59101]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:07.024 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59101]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:07.024 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:06 smithi102 sudo[59101]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:07.024 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59157]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:07.033 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:00:07.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:00:07.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:06 smithi152 ceph-mon[33523]: osdmap e20: 3 total, 3 up, 3 in 2024-04-10T03:00:07.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:07.275 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59157]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:07.275 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59157]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:07.275 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59157]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:07.537 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59183]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:07.537 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59183]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:07.537 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59183]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:07.538 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59183]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59206]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59206]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59206]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:07.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59202]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:07.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59202]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:07.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59202]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:07.793 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59202]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[39651]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[39651]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[39651]: osdmap e21: 3 total, 3 up, 3 in 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:07 smithi102 sudo[59206]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59209]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59209]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:08.129 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59209]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:08.130 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59209]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:08.130 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[29301]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:08.130 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[29301]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-10T03:00:08.130 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[29301]: osdmap e21: 3 total, 3 up, 3 in 2024-04-10T03:00:08.130 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:08 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:08.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 ceph-mon[33523]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:08.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 ceph-mon[33523]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-10T03:00:08.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 ceph-mon[33523]: osdmap e21: 3 total, 3 up, 3 in 2024-04-10T03:00:08.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:08.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36083]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:08.302 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36083]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:08.302 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36083]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:08.302 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36083]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36086]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36086]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36086]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:08 smithi152 sudo[36086]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59259]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-10T03:00:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59259]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59259]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:08 smithi102 sudo[59259]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.371 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:09.372 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 sudo[59336]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 sudo[59336]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 sudo[59336]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 sudo[59336]: pam_unix(sudo:session): session closed for user root 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:09.373 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:09.374 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:09.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:00:09.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:09.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:09.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:09.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:10.198 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:10 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:10.198 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:10 smithi102 ceph-mon[39651]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:10.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:10 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:10.199 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:10 smithi102 ceph-mon[29301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:10.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:10 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-10T03:00:10.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:10 smithi152 ceph-mon[33523]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-10T03:00:11.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:11 smithi152 ceph-mon[33523]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:11.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:11 smithi102 ceph-mon[29301]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:11.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:11 smithi102 ceph-mon[39651]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/590567606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]': finished 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: osdmap e22: 4 total, 3 up, 4 in 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:12.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:12 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/219376403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:12.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/590567606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]': finished 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: osdmap e22: 4 total, 3 up, 4 in 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/219376403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/590567606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]: dispatch 2024-04-10T03:00:12.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "152f506d-f4fc-44d1-8252-0b994b129f90"}]': finished 2024-04-10T03:00:12.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: osdmap e22: 4 total, 3 up, 4 in 2024-04-10T03:00:12.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:12.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:12 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/219376403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:13.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:13 smithi152 ceph-mon[33523]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:13 smithi102 ceph-mon[29301]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:13.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:13 smithi102 ceph-mon[39651]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:15.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:15 smithi102 ceph-mon[29301]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:15.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:15 smithi102 ceph-mon[39651]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:16.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:15 smithi152 ceph-mon[33523]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:17.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:17 smithi102 ceph-mon[29301]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:17.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:17 smithi102 ceph-mon[39651]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:18.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:17 smithi152 ceph-mon[33523]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:00:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[29301]: Deploying daemon osd.3 on smithi102 2024-04-10T03:00:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:00:18.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:18.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:18 smithi102 ceph-mon[39651]: Deploying daemon osd.3 on smithi102 2024-04-10T03:00:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:18 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:00:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:18 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:18 smithi152 ceph-mon[33523]: Deploying daemon osd.3 on smithi102 2024-04-10T03:00:19.551 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:19 smithi102 ceph-mon[29301]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:19.552 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:19 smithi102 ceph-mon[39651]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:20.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:19 smithi152 ceph-mon[33523]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:21.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:21 smithi102 ceph-mon[29301]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:21.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:21 smithi102 ceph-mon[39651]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:22.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:21 smithi152 ceph-mon[33523]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:24.230 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[29301]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[39651]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.231 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:23 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:23 smithi152 ceph-mon[33523]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:23 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:23 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:23 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:25.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[29301]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:25.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:25.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:25.984 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[39651]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:25.984 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:25.984 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:25 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:25 smithi152 ceph-mon[33523]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:26.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:25 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:25 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.366 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 3 on host 'smithi102' 2024-04-10T03:00:26.946 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.947 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:26.948 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:26 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:26.971 DEBUG:teuthology.orchestra.run.smithi102:osd.3> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.3.service 2024-04-10T03:00:26.974 INFO:tasks.cephadm:Deploying osd.4 on smithi152 with /dev/vg_nvme/lv_4... 2024-04-10T03:00:26.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-10T03:00:27.231 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:00:27.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:27.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:26 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[29301]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[29301]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[29301]: osdmap e23: 4 total, 3 up, 4 in 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[29301]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[39651]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[39651]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:00:28.170 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[39651]: osdmap e23: 4 total, 3 up, 4 in 2024-04-10T03:00:28.171 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:28.171 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-mon[39651]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:28.171 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[61165]: 2024-04-10T03:00:27.910+0000 7eff46086700 -1 osd.3 0 waiting for initial osdmap 2024-04-10T03:00:28.171 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:00:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[61165]: 2024-04-10T03:00:27.918+0000 7eff3ee7d700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:00:28.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:27 smithi152 ceph-mon[33523]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:28.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:27 smithi152 ceph-mon[33523]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:00:28.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:27 smithi152 ceph-mon[33523]: osdmap e23: 4 total, 3 up, 4 in 2024-04-10T03:00:28.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:27 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:28.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:27 smithi152 ceph-mon[33523]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0116444 s, 901 MB/s 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 2024-04-10T03:00:29.019 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-10T03:00:29.234 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[29301]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[29301]: osdmap e24: 4 total, 3 up, 4 in 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[39651]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[39651]: osdmap e24: 4 total, 3 up, 4 in 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.235 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:28 smithi152 ceph-mon[33523]: from='osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-04-10T03:00:29.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:28 smithi152 ceph-mon[33523]: osdmap e24: 4 total, 3 up, 4 in 2024-04-10T03:00:29.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:29.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_4 2024-04-10T03:00:29.941 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932] boot 2024-04-10T03:00:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: osdmap e25: 4 total, 4 up, 4 in 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932] boot 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: osdmap e25: 4 total, 4 up, 4 in 2024-04-10T03:00:30.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:29 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: osd.3 [v2:172.21.15.102:6826/248402932,v1:172.21.15.102:6827/248402932] boot 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: osdmap e25: 4 total, 4 up, 4 in 2024-04-10T03:00:30.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:29 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: osdmap e26: 4 total, 4 up, 4 in 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:31.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:31.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:30 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: osdmap e26: 4 total, 4 up, 4 in 2024-04-10T03:00:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: osdmap e26: 4 total, 4 up, 4 in 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:31.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:30 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi102 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi102 to 4016M 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:32.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:31 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:32.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi102 2024-04-10T03:00:32.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi102 to 4016M 2024-04-10T03:00:32.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:32.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:32.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:32.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:32.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi102 2024-04-10T03:00:32.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi102 to 4016M 2024-04-10T03:00:32.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:32.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:32.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:32.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:31 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:33.193 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:32 smithi102 ceph-mon[39651]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:33.194 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:32 smithi102 ceph-mon[29301]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:33.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:32 smithi152 ceph-mon[33523]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:34.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:33 smithi152 ceph-mon[33523]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:34.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:33 smithi102 ceph-mon[29301]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:34.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:33 smithi102 ceph-mon[39651]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:35.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/3894287138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]': finished 2024-04-10T03:00:35.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: osdmap e27: 5 total, 4 up, 5 in 2024-04-10T03:00:35.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:35.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:34 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/3863170496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/3894287138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]': finished 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: osdmap e27: 5 total, 4 up, 5 in 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/3863170496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/3894287138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]: dispatch 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "382c51f3-c988-48aa-9f0d-3ad491ad17cb"}]': finished 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: osdmap e27: 5 total, 4 up, 5 in 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:34 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/3863170496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:36.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:35 smithi152 ceph-mon[33523]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:36.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:35 smithi102 ceph-mon[39651]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:36.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:35 smithi102 ceph-mon[29301]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:38.159 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:37 smithi152 ceph-mon[33523]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:38.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:37 smithi102 ceph-mon[39651]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:38.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:37 smithi102 ceph-mon[29301]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:39 smithi152 ceph-mon[33523]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:39 smithi102 ceph-mon[29301]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:40.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:39 smithi102 ceph-mon[39651]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:41.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:40 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:00:41.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:40 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:41.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:40 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:00:41.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:40 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:41.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:40 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:00:41.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:40 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:41 smithi152 ceph-mon[33523]: Deploying daemon osd.4 on smithi152 2024-04-10T03:00:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:41 smithi152 ceph-mon[33523]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:42.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:41 smithi102 ceph-mon[39651]: Deploying daemon osd.4 on smithi152 2024-04-10T03:00:42.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:41 smithi102 ceph-mon[39651]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:42.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:41 smithi102 ceph-mon[29301]: Deploying daemon osd.4 on smithi152 2024-04-10T03:00:42.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:41 smithi102 ceph-mon[29301]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:44.258 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:43 smithi152 ceph-mon[33523]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:43 smithi102 ceph-mon[29301]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:44.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:43 smithi102 ceph-mon[39651]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:46.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:45 smithi152 ceph-mon[33523]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:46.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[29301]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:46.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[39651]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:46.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:46.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:48.105 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:48.105 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:48.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:48.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:48.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:48.441 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 4 on host 'smithi152' 2024-04-10T03:00:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:48 smithi152 ceph-mon[33523]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:48 smithi152 ceph-mon[33523]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:48 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:48 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:48 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:49.083 DEBUG:teuthology.orchestra.run.smithi152:osd.4> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.4.service 2024-04-10T03:00:49.086 INFO:tasks.cephadm:Deploying osd.5 on smithi152 with /dev/vg_nvme/lv_3... 2024-04-10T03:00:49.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-10T03:00:49.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[29301]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[29301]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[39651]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[39651]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:49.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:48 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:00:50.233 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:50.233 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-10T03:00:50.233 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: osdmap e28: 5 total, 4 up, 5 in 2024-04-10T03:00:50.234 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:50.234 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.234 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-mon[33523]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.234 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:00:49.966+0000 7f580c3ea700 -1 osd.4 0 waiting for initial osdmap 2024-04-10T03:00:50.234 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:00:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:00:49.970+0000 7f58079e6700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:00:50.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:50.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-10T03:00:50.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: osdmap e28: 5 total, 4 up, 5 in 2024-04-10T03:00:50.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:50.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[29301]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: osdmap e28: 5 total, 4 up, 5 in 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: from='osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:49 smithi102 ceph-mon[39651]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:00:50.734 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: osdmap e29: 5 total, 4 up, 5 in 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292] boot 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: osdmap e30: 5 total, 5 up, 5 in 2024-04-10T03:00:51.099 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:50 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:00:51.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: osdmap e29: 5 total, 4 up, 5 in 2024-04-10T03:00:51.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292] boot 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: osdmap e30: 5 total, 5 up, 5 in 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: osdmap e29: 5 total, 4 up, 5 in 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: osd.4 [v2:172.21.15.152:6800/1841668292,v1:172.21.15.152:6801/1841668292] boot 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: osdmap e30: 5 total, 5 up, 5 in 2024-04-10T03:00:51.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:50 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:00:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:51 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:00:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:51 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:00:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:51 smithi152 ceph-mon[33523]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:51 smithi152 ceph-mon[33523]: osdmap e31: 5 total, 5 up, 5 in 2024-04-10T03:00:52.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:00:52.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[29301]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[29301]: osdmap e31: 5 total, 5 up, 5 in 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[39651]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-10T03:00:52.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:51 smithi102 ceph-mon[39651]: osdmap e31: 5 total, 5 up, 5 in 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0115958 s, 904 MB/s 2024-04-10T03:00:53.245 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-10T03:00:53.245 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi152 2024-04-10T03:00:53.245 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.245 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 17088M 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: osdmap e32: 5 total, 5 up, 5 in 2024-04-10T03:00:53.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:53 smithi152 ceph-mon[33523]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi152 2024-04-10T03:00:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 17088M 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: osdmap e32: 5 total, 5 up, 5 in 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[29301]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi152 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 17088M 2024-04-10T03:00:53.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:53.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:00:53.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:00:53.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: osdmap e32: 5 total, 5 up, 5 in 2024-04-10T03:00:53.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:53 smithi102 ceph-mon[39651]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:53.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_3 2024-04-10T03:00:54.155 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:00:54.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:54 smithi152 ceph-mon[33523]: osdmap e33: 5 total, 5 up, 5 in 2024-04-10T03:00:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:54 smithi102 ceph-mon[29301]: osdmap e33: 5 total, 5 up, 5 in 2024-04-10T03:00:54.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:54 smithi102 ceph-mon[39651]: osdmap e33: 5 total, 5 up, 5 in 2024-04-10T03:00:55.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:55 smithi152 ceph-mon[33523]: osdmap e34: 5 total, 5 up, 5 in 2024-04-10T03:00:55.440 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:55 smithi152 ceph-mon[33523]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:55 smithi102 ceph-mon[29301]: osdmap e34: 5 total, 5 up, 5 in 2024-04-10T03:00:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:55 smithi102 ceph-mon[29301]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:55 smithi102 ceph-mon[39651]: osdmap e34: 5 total, 5 up, 5 in 2024-04-10T03:00:55.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:55 smithi102 ceph-mon[39651]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:00:56.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:56 smithi152 ceph-mon[33523]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:56.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:56 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:56.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:56 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:56.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:56 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:56.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[29301]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:56.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:56.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:56.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:56.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[39651]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:00:56.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:00:56.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:00:56.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:56 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:00:57.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:57 smithi152 ceph-mon[33523]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:00:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:57 smithi102 ceph-mon[29301]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:00:57.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:57 smithi102 ceph-mon[39651]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:00:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:58 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/2483591644' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:58 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:58 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]': finished 2024-04-10T03:00:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:58 smithi152 ceph-mon[33523]: osdmap e35: 6 total, 5 up, 6 in 2024-04-10T03:00:58.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:58 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:00:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/2483591644' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]': finished 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[29301]: osdmap e35: 6 total, 5 up, 6 in 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/2483591644' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]: dispatch 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef"}]': finished 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[39651]: osdmap e35: 6 total, 5 up, 6 in 2024-04-10T03:00:58.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:58 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:00:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:59 smithi152 ceph-mon[33523]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:00:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:00:59 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/1568953989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:59 smithi102 ceph-mon[29301]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:00:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:00:59 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/1568953989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:00:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:59 smithi102 ceph-mon[39651]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:00:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:00:59 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/1568953989' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:01 smithi152 ceph-mon[33523]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-10T03:01:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:01 smithi102 ceph-mon[29301]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-10T03:01:01.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:01 smithi102 ceph-mon[39651]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-10T03:01:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:03 smithi102 ceph-mon[29301]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-10T03:01:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:03 smithi102 ceph-mon[39651]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-10T03:01:03.932 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:03 smithi152 ceph-mon[33523]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-10T03:01:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:01:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:01:04.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:04.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:01:04.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:05.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:05 smithi152 ceph-mon[33523]: Deploying daemon osd.5 on smithi152 2024-04-10T03:01:05.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:05 smithi152 ceph-mon[33523]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-10T03:01:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:05 smithi102 ceph-mon[29301]: Deploying daemon osd.5 on smithi152 2024-04-10T03:01:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:05 smithi102 ceph-mon[29301]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-10T03:01:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:05 smithi102 ceph-mon[39651]: Deploying daemon osd.5 on smithi152 2024-04-10T03:01:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:05 smithi102 ceph-mon[39651]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-10T03:01:07.663 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:07 smithi152 ceph-mon[33523]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:07 smithi102 ceph-mon[29301]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:07 smithi102 ceph-mon[39651]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[29301]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[39651]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:09.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:09 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:09.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:09 smithi152 ceph-mon[33523]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:09.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:09.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:09.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:09 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:11.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:11 smithi152 ceph-mon[33523]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:11 smithi102 ceph-mon[29301]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:11 smithi102 ceph-mon[39651]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:12.307 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 5 on host 'smithi152' 2024-04-10T03:01:12.958 DEBUG:teuthology.orchestra.run.smithi152:osd.5> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.5.service 2024-04-10T03:01:12.960 INFO:tasks.cephadm:Deploying osd.6 on smithi152 with /dev/vg_nvme/lv_2... 2024-04-10T03:01:12.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-10T03:01:13.256 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:01:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:13.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:14.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-mon[33523]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-10T03:01:14.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-mon[33523]: osdmap e36: 6 total, 5 up, 6 in 2024-04-10T03:01:14.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:14.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-mon[33523]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:14.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-mon[33523]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:14.508 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:01:14.302+0000 7f83dbbc3700 -1 osd.5 0 waiting for initial osdmap 2024-04-10T03:01:14.508 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:01:14 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:01:14.308+0000 7f83d81c1700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[29301]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[29301]: osdmap e36: 6 total, 5 up, 6 in 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[29301]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[29301]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[39651]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[39651]: osdmap e36: 6 total, 5 up, 6 in 2024-04-10T03:01:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:14.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[39651]: from='osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:14.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:14 smithi102 ceph-mon[39651]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012486 s, 840 MB/s 2024-04-10T03:01:15.085 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: osdmap e37: 6 total, 5 up, 6 in 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863] boot 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: osdmap e38: 6 total, 6 up, 6 in 2024-04-10T03:01:15.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:15 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: osdmap e37: 6 total, 5 up, 6 in 2024-04-10T03:01:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863] boot 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: osdmap e38: 6 total, 6 up, 6 in 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: osdmap e37: 6 total, 5 up, 6 in 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: osd.5 [v2:172.21.15.152:6808/537038863,v1:172.21.15.152:6809/537038863] boot 2024-04-10T03:01:15.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: osdmap e38: 6 total, 6 up, 6 in 2024-04-10T03:01:15.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:15 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:01:16.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:16 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:01:16.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:16 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:01:16.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:16 smithi152 ceph-mon[33523]: osdmap e39: 6 total, 6 up, 6 in 2024-04-10T03:01:16.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_2 2024-04-10T03:01:16.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:01:16.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:01:16.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[29301]: osdmap e39: 6 total, 6 up, 6 in 2024-04-10T03:01:16.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:01:16.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:01:16.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:16 smithi102 ceph-mon[39651]: osdmap e39: 6 total, 6 up, 6 in 2024-04-10T03:01:16.994 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:01:17.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:17 smithi152 ceph-mon[33523]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:17.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:17 smithi152 ceph-mon[33523]: osdmap e40: 6 total, 6 up, 6 in 2024-04-10T03:01:17.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:17 smithi102 ceph-mon[29301]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:17.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:17 smithi102 ceph-mon[29301]: osdmap e40: 6 total, 6 up, 6 in 2024-04-10T03:01:17.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:17 smithi102 ceph-mon[39651]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:17.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:17 smithi102 ceph-mon[39651]: osdmap e40: 6 total, 6 up, 6 in 2024-04-10T03:01:18.720 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:18 smithi152 ceph-mon[33523]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:01:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:18 smithi102 ceph-mon[29301]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:01:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:18 smithi102 ceph-mon[39651]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:01:19.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: osdmap e41: 6 total, 6 up, 6 in 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi152 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 8544M 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:19.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.765 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.765 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:19 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: osdmap e41: 6 total, 6 up, 6 in 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi152 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 8544M 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: osdmap e41: 6 total, 6 up, 6 in 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi152 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 8544M 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:19.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:19 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:20.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:20 smithi152 ceph-mon[33523]: osdmap e42: 6 total, 6 up, 6 in 2024-04-10T03:01:20.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:20 smithi152 ceph-mon[33523]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:20.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:20.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:20.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:20 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[29301]: osdmap e42: 6 total, 6 up, 6 in 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[29301]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:20.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[39651]: osdmap e42: 6 total, 6 up, 6 in 2024-04-10T03:01:20.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[39651]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:20.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:20.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:20.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:20 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:21.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:21 smithi152 ceph-mon[33523]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:21.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:21 smithi102 ceph-mon[29301]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:21.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:21 smithi102 ceph-mon[39651]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/1284966101' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]': finished 2024-04-10T03:01:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: osdmap e43: 7 total, 6 up, 7 in 2024-04-10T03:01:22.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:22 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: Cluster is now healthy 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/1284966101' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]': finished 2024-04-10T03:01:22.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: osdmap e43: 7 total, 6 up, 7 in 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/1284966101' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]: dispatch 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64f5afa7-c8f8-448d-a0c7-e220655282db"}]': finished 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: osdmap e43: 7 total, 6 up, 7 in 2024-04-10T03:01:22.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:22 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:23.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:23 smithi152 ceph-mon[33523]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:01:23.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:23 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/858905194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:23.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:23 smithi102 ceph-mon[29301]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:01:23.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:23 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/858905194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:23.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:23 smithi102 ceph-mon[39651]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:01:23.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:23 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/858905194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:25.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:25 smithi152 ceph-mon[33523]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-10T03:01:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:25 smithi102 ceph-mon[29301]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-10T03:01:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:25 smithi102 ceph-mon[39651]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-10T03:01:27.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:27 smithi102 ceph-mon[29301]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-10T03:01:27.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:27 smithi102 ceph-mon[39651]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-10T03:01:27.846 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:27 smithi152 ceph-mon[33523]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-10T03:01:28.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:01:28.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:01:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:29.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:01:29.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:29.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:29 smithi152 ceph-mon[33523]: Deploying daemon osd.6 on smithi152 2024-04-10T03:01:29.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:29 smithi152 ceph-mon[33523]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-10T03:01:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:29 smithi102 ceph-mon[29301]: Deploying daemon osd.6 on smithi152 2024-04-10T03:01:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:29 smithi102 ceph-mon[29301]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-10T03:01:30.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:29 smithi102 ceph-mon[39651]: Deploying daemon osd.6 on smithi152 2024-04-10T03:01:30.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:29 smithi102 ceph-mon[39651]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-10T03:01:31.875 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:31 smithi152 ceph-mon[33523]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:32.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:31 smithi102 ceph-mon[29301]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:32.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:31 smithi102 ceph-mon[39651]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:33 smithi152 ceph-mon[33523]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:33.843 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:33 smithi102 ceph-mon[29301]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:33.843 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:33 smithi102 ceph-mon[39651]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:34.907 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:34 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:34.907 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:34 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:34.907 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:34 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:35.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:35.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:35.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:34 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:35 smithi152 ceph-mon[33523]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:36.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:35 smithi102 ceph-mon[39651]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:35 smithi102 ceph-mon[29301]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:36.423 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 6 on host 'smithi152' 2024-04-10T03:01:37.322 DEBUG:teuthology.orchestra.run.smithi152:osd.6> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.6.service 2024-04-10T03:01:37.325 INFO:tasks.cephadm:Deploying osd.7 on smithi152 with /dev/vg_nvme/lv_1... 2024-04-10T03:01:37.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-10T03:01:37.629 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:01:37.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:37 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:37 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:37 smithi152 ceph-mon[33523]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:37.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:37 smithi152 ceph-mon[33523]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:37.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:37 smithi152 ceph-mon[33523]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[29301]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[29301]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[29301]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[39651]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[39651]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:37.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:37 smithi102 ceph-mon[39651]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: osdmap e44: 7 total, 6 up, 7 in 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:38.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:38.749 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:01:38.414+0000 7f9b2838e700 -1 osd.6 0 waiting for initial osdmap 2024-04-10T03:01:38.749 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:01:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:01:38.420+0000 7f9b23189700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: osdmap e44: 7 total, 6 up, 7 in 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: osdmap e44: 7 total, 6 up, 7 in 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:38.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:38 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:39.650 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-10T03:01:39.650 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-10T03:01:39.650 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-10T03:01:39.650 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-10T03:01:39.651 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124022 s, 845 MB/s 2024-04-10T03:01:39.651 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-10T03:01:39.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:39 smithi152 ceph-mon[33523]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:39.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:39 smithi152 ceph-mon[33523]: osdmap e45: 7 total, 6 up, 7 in 2024-04-10T03:01:39.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:39 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:39 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:39 smithi152 ceph-mon[33523]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:39.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[29301]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:39.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[29301]: osdmap e45: 7 total, 6 up, 7 in 2024-04-10T03:01:39.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[29301]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[39651]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:01:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[39651]: osdmap e45: 7 total, 6 up, 7 in 2024-04-10T03:01:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:39.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:39 smithi102 ceph-mon[39651]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-10T03:01:40.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:01:40.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:01:40.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471] boot 2024-04-10T03:01:40.636 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: osdmap e46: 7 total, 7 up, 7 in 2024-04-10T03:01:40.636 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:40.636 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:40 smithi152 ceph-mon[33523]: osdmap e47: 7 total, 7 up, 7 in 2024-04-10T03:01:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:01:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471] boot 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: osdmap e46: 7 total, 7 up, 7 in 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[29301]: osdmap e47: 7 total, 7 up, 7 in 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: osd.6 [v2:172.21.15.152:6816/524183471,v1:172.21.15.152:6817/524183471] boot 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: osdmap e46: 7 total, 7 up, 7 in 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:01:40.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:40 smithi102 ceph-mon[39651]: osdmap e47: 7 total, 7 up, 7 in 2024-04-10T03:01:41.609 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:41 smithi152 ceph-mon[33523]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:41.609 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:41 smithi152 ceph-mon[33523]: osdmap e48: 7 total, 7 up, 7 in 2024-04-10T03:01:41.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_1 2024-04-10T03:01:41.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:41 smithi102 ceph-mon[39651]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:41.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:41 smithi102 ceph-mon[39651]: osdmap e48: 7 total, 7 up, 7 in 2024-04-10T03:01:41.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:41 smithi102 ceph-mon[29301]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:41.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:41 smithi102 ceph-mon[29301]: osdmap e48: 7 total, 7 up, 7 in 2024-04-10T03:01:42.532 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: osdmap e49: 7 total, 7 up, 7 in 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi152 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.630 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 5696M 2024-04-10T03:01:43.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-10T03:01:43.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:43.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.633 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:43 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: osdmap e49: 7 total, 7 up, 7 in 2024-04-10T03:01:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi152 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 5696M 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-10T03:01:43.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: osdmap e49: 7 total, 7 up, 7 in 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi152 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 5696M 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:01:43.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:43 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:44.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:44 smithi152 ceph-mon[33523]: osdmap e50: 7 total, 7 up, 7 in 2024-04-10T03:01:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:44 smithi102 ceph-mon[29301]: osdmap e50: 7 total, 7 up, 7 in 2024-04-10T03:01:44.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:44 smithi102 ceph-mon[39651]: osdmap e50: 7 total, 7 up, 7 in 2024-04-10T03:01:45.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:45 smithi152 ceph-mon[33523]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:45.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:45.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:45.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:45 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:45.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:45 smithi152 ceph-mon[33523]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-04-10T03:01:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[39651]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[39651]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-04-10T03:01:45.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[29301]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:01:45.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-10T03:01:45.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-10T03:01:45.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:45.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:45 smithi102 ceph-mon[29301]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-04-10T03:01:47.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/3176841397' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]': finished 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: osdmap e51: 8 total, 7 up, 8 in 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/2107237873' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/3176841397' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]': finished 2024-04-10T03:01:47.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: osdmap e51: 8 total, 7 up, 8 in 2024-04-10T03:01:47.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:01:47.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:47 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/2107237873' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/3176841397' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]: dispatch 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0d07434-ccb1-4d71-92aa-bda52e0b6edb"}]': finished 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: osdmap e51: 8 total, 7 up, 8 in 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:01:48.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:47 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/2107237873' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-10T03:01:49.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:49 smithi102 ceph-mon[29301]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:49 smithi102 ceph-mon[39651]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:49 smithi152 ceph-mon[33523]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:51.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:51 smithi102 ceph-mon[29301]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:51.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:51 smithi102 ceph-mon[39651]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:51.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:51 smithi152 ceph-mon[33523]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:53.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:53 smithi152 ceph-mon[33523]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:53.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:01:53.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:53 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[29301]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:01:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[39651]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:01:54.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:53 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:01:54.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:54 smithi152 ceph-mon[33523]: Deploying daemon osd.7 on smithi152 2024-04-10T03:01:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:54 smithi102 ceph-mon[29301]: Deploying daemon osd.7 on smithi152 2024-04-10T03:01:55.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:54 smithi102 ceph-mon[39651]: Deploying daemon osd.7 on smithi152 2024-04-10T03:01:55.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:55 smithi152 ceph-mon[33523]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:56.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:55 smithi102 ceph-mon[29301]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:56.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:55 smithi102 ceph-mon[39651]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:57.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:57 smithi152 ceph-mon[33523]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:57 smithi102 ceph-mon[29301]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:58.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:57 smithi102 ceph-mon[39651]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:59 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:59 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:59 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:01:59 smithi152 ceph-mon[33523]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[29301]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:01:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:01:59.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:01:59 smithi102 ceph-mon[39651]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:01.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:01.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:01.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:01.680 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:01 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:01 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:01.993 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 7 on host 'smithi152' 2024-04-10T03:02:02.634 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:02 smithi152 ceph-mon[33523]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:02.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:02 smithi152 ceph-mon[33523]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:02.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:02 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:02.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:02 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:02.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:02 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:02.663 DEBUG:teuthology.orchestra.run.smithi152:osd.7> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.7.service 2024-04-10T03:02:02.665 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-10T03:02:02.665 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd stat -f json 2024-04-10T03:02:02.932 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[39651]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[39651]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[29301]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[29301]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:02 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:03.557 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:03.557 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-10T03:02:03.558 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: osdmap e52: 8 total, 7 up, 8 in 2024-04-10T03:02:03.558 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:03.558 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:03.558 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[29301]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: osdmap e52: 8 total, 7 up, 8 in 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:03.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-mon[33523]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:03.814 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:02:03.562+0000 7fbeb52e0700 -1 osd.7 0 waiting for initial osdmap 2024-04-10T03:02:03.814 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:02:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:02:03.570+0000 7fbeaf0d9700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: osdmap e52: 8 total, 7 up, 8 in 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: from='osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:03 smithi102 ceph-mon[39651]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: osdmap e53: 8 total, 7 up, 8 in 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160] boot 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: osdmap e54: 8 total, 8 up, 8 in 2024-04-10T03:02:04.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: osdmap e53: 8 total, 7 up, 8 in 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160] boot 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: osdmap e54: 8 total, 8 up, 8 in 2024-04-10T03:02:04.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:04 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.825 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: osdmap e53: 8 total, 7 up, 8 in 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: osd.7 [v2:172.21.15.152:6824/2303235160,v1:172.21.15.152:6825/2303235160] boot 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: osdmap e54: 8 total, 8 up, 8 in 2024-04-10T03:02:04.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:04 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:05.565 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1712718124,"num_in_osds":8,"osd_in_since":1712718106,"num_remapped_pgs":1} 2024-04-10T03:02:05.565 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[39651]: purged_snaps scrub starts 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[39651]: purged_snaps scrub ok 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[39651]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/528681239' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[39651]: osdmap e55: 8 total, 8 up, 8 in 2024-04-10T03:02:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[29301]: purged_snaps scrub starts 2024-04-10T03:02:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[29301]: purged_snaps scrub ok 2024-04-10T03:02:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[29301]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/528681239' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-10T03:02:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:05 smithi102 ceph-mon[29301]: osdmap e55: 8 total, 8 up, 8 in 2024-04-10T03:02:05.836 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:05.849 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:05 smithi152 ceph-mon[33523]: purged_snaps scrub starts 2024-04-10T03:02:05.849 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:05 smithi152 ceph-mon[33523]: purged_snaps scrub ok 2024-04-10T03:02:05.849 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:05 smithi152 ceph-mon[33523]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-10T03:02:05.849 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:05 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/528681239' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-10T03:02:05.849 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:05 smithi152 ceph-mon[33523]: osdmap e55: 8 total, 8 up, 8 in 2024-04-10T03:02:06.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: Detected new or changed devices on smithi152 2024-04-10T03:02:06.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 4272M 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[39651]: osdmap e56: 8 total, 8 up, 8 in 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: Detected new or changed devices on smithi152 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 4272M 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:06.925 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:06 smithi102 ceph-mon[29301]: osdmap e56: 8 total, 8 up, 8 in 2024-04-10T03:02:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: Detected new or changed devices on smithi152 2024-04-10T03:02:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 4272M 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:06 smithi152 ceph-mon[33523]: osdmap e56: 8 total, 8 up, 8 in 2024-04-10T03:02:07.612 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:02:07.613 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":57,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:02:07.296771+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:02:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:07 smithi152 ceph-mon[33523]: pgmap v172: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s, 0 objects/s recovering 2024-04-10T03:02:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:07 smithi152 ceph-mon[33523]: osdmap e57: 8 total, 8 up, 8 in 2024-04-10T03:02:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:07 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/955393364' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[29301]: pgmap v172: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s, 0 objects/s recovering 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[29301]: osdmap e57: 8 total, 8 up, 8 in 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/955393364' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[39651]: pgmap v172: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s, 0 objects/s recovering 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[39651]: osdmap e57: 8 total, 8 up, 8 in 2024-04-10T03:02:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:07 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/955393364' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:08.338 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-10T03:00:04.753220+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-10T03:02:08.338 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-10T03:02:08.588 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:09.704 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:09 smithi102 ceph-mon[29301]: osdmap e58: 8 total, 8 up, 8 in 2024-04-10T03:02:09.704 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:09 smithi102 ceph-mon[29301]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s, 0 objects/s recovering 2024-04-10T03:02:09.705 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:09 smithi102 ceph-mon[39651]: osdmap e58: 8 total, 8 up, 8 in 2024-04-10T03:02:09.705 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:09 smithi102 ceph-mon[39651]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s, 0 objects/s recovering 2024-04-10T03:02:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:09 smithi152 ceph-mon[33523]: osdmap e58: 8 total, 8 up, 8 in 2024-04-10T03:02:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:09 smithi152 ceph-mon[33523]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s, 0 objects/s recovering 2024-04-10T03:02:10.413 INFO:teuthology.orchestra.run.smithi102.stdout:pg_num: 1 2024-04-10T03:02:10.737 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:10 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3028848674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-10T03:02:10.737 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:10 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3028848674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-10T03:02:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:10 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3028848674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-10T03:02:11.112 INFO:tasks.cephadm:Adding prometheus.a on smithi152 2024-04-10T03:02:11.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi152=a' 2024-04-10T03:02:11.380 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:02:11.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:11 smithi152 ceph-mon[33523]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:02:11.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:11 smithi102 ceph-mon[29301]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:02:11.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:11 smithi102 ceph-mon[39651]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-10T03:02:13.225 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled prometheus update... 2024-04-10T03:02:13.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-10T03:02:13.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:13.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:13.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:13.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:13.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:13 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:13.891 DEBUG:teuthology.orchestra.run.smithi152:prometheus.a> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service 2024-04-10T03:02:13.893 INFO:tasks.cephadm:Adding node-exporter.a on smithi102 2024-04-10T03:02:13.893 INFO:tasks.cephadm:Adding node-exporter.b on smithi152 2024-04-10T03:02:13.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi102=a;smithi152=b' 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:13 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:14.207 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:02:14.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:14 smithi152 ceph-mon[33523]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:14.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:14 smithi152 ceph-mon[33523]: Saving service prometheus spec with placement smithi152=a;count:1 2024-04-10T03:02:14.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:14 smithi152 ceph-mon[33523]: Deploying daemon prometheus.a on smithi152 2024-04-10T03:02:15.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[29301]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:15.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[29301]: Saving service prometheus spec with placement smithi152=a;count:1 2024-04-10T03:02:15.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[29301]: Deploying daemon prometheus.a on smithi152 2024-04-10T03:02:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[39651]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[39651]: Saving service prometheus spec with placement smithi152=a;count:1 2024-04-10T03:02:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:14 smithi102 ceph-mon[39651]: Deploying daemon prometheus.a on smithi152 2024-04-10T03:02:15.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:15 smithi152 ceph-mon[33523]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:15 smithi102 ceph-mon[29301]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:16.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:15 smithi102 ceph-mon[39651]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:16.818 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled node-exporter update... 2024-04-10T03:02:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:17 smithi152 ceph-mon[33523]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:17 smithi152 ceph-mon[33523]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi102=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:17 smithi152 ceph-mon[33523]: Saving service node-exporter spec with placement smithi102=a;smithi152=b;count:2 2024-04-10T03:02:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:17 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:18.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[29301]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:18.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[29301]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi102=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:18.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[29301]: Saving service node-exporter spec with placement smithi102=a;smithi152=b;count:2 2024-04-10T03:02:18.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:18.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[39651]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:18.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[39651]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi102=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:18.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[39651]: Saving service node-exporter spec with placement smithi102=a;smithi152=b;count:2 2024-04-10T03:02:18.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:17 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:19.246 DEBUG:teuthology.orchestra.run.smithi102:node-exporter.a> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service 2024-04-10T03:02:19.249 DEBUG:teuthology.orchestra.run.smithi152:node-exporter.b> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service 2024-04-10T03:02:19.251 INFO:tasks.cephadm:Adding alertmanager.a on smithi102 2024-04-10T03:02:19.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi102=a' 2024-04-10T03:02:19.539 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:02:20.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:19 smithi152 ceph-mon[33523]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:20.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:19 smithi102 ceph-mon[29301]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:20.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:19 smithi102 ceph-mon[39651]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:22.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:21 smithi152 ceph-mon[33523]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:22.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:21 smithi102 ceph-mon[29301]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:22.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:21 smithi102 ceph-mon[39651]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:22.329 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled alertmanager update... 2024-04-10T03:02:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:23 smithi152 ceph-mon[33523]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:23.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:23 smithi152 ceph-mon[33523]: Saving service alertmanager spec with placement smithi102=a;count:1 2024-04-10T03:02:23.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:23 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:23.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:23 smithi152 ceph-mon[33523]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:23.600 DEBUG:teuthology.orchestra.run.smithi102:alertmanager.a> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service 2024-04-10T03:02:23.603 INFO:tasks.cephadm:Adding grafana.a on smithi152 2024-04-10T03:02:23.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph orch apply grafana '1;smithi152=a' 2024-04-10T03:02:23.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[39651]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:23.628 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[39651]: Saving service alertmanager spec with placement smithi102=a;count:1 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[39651]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[29301]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[29301]: Saving service alertmanager spec with placement smithi102=a;count:1 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:23.629 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:23 smithi102 ceph-mon[29301]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:25.561 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:02:25.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:25 smithi152 ceph-mon[33523]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:25 smithi102 ceph-mon[29301]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:26.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:25 smithi102 ceph-mon[39651]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:27.487 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled grafana update... 2024-04-10T03:02:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:27 smithi152 ceph-mon[33523]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:27 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:27 smithi102 ceph-mon[29301]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:27 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:27 smithi102 ceph-mon[39651]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:27 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.089 DEBUG:teuthology.orchestra.run.smithi152:grafana.a> sudo journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service 2024-04-10T03:02:28.092 INFO:tasks.cephadm:Setting up client nodes... 2024-04-10T03:02:28.092 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-10T03:02:28.209 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 podman[60769]: 2024-04-10 03:02:28.188621423 +0000 UTC m=+1.123941418 container start 8a1b92be2198d925bb6ca60e0ea402a98e429ce2d1dc7171e414c8b2f212bee0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:02:28.352 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:28.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.212Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.212Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.212Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi152 (none))" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.212Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.212Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.213Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.214Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.215Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.215Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.217Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.217Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.934µs 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.217Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-10T03:02:28.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.217Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.217Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.551µs wal_replay_duration=151.339µs wbl_replay_duration=107ns total_replay_duration=188.877µs 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.218Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.218Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.218Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.232Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.597029ms db_storage=660ns remote_storage=1.061µs web_handler=324ns query_engine=502ns scrape=371.17µs scrape_sd=16.26µs notify=946ns notify_sd=1.746µs rules=14.048353ms tracing=13.94µs 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.232Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:02:28.232Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 bash[60769]: 8a1b92be2198d925bb6ca60e0ea402a98e429ce2d1dc7171e414c8b2f212bee0 2024-04-10T03:02:28.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:02:28 smithi152 systemd[1]: Started Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: Saving service grafana spec with placement smithi152=a;count:1 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: Saving service grafana spec with placement smithi152=a;count:1 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:28.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:28 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: Saving service grafana spec with placement smithi152=a;count:1 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' 2024-04-10T03:02:29.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:28 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-10T03:02:29.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[29301]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:29.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[29301]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-10T03:02:29.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[29301]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-10T03:02:29.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[39651]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:29.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[39651]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-10T03:02:29.728 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mon[39651]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-10T03:02:29.728 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setuser ceph since I am not root 2024-04-10T03:02:29.728 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setgroup ceph since I am not root 2024-04-10T03:02:29.728 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-mgr[29542]: -- 172.21.15.102:0/3503710804 <== mon.0 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562147b109c0 con 0x562147ae6800 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-mon[33523]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-mon[33523]: from='mgr.14150 172.21.15.102:0/1088801469' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-mon[33523]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ignoring --setuser ceph since I am not root 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ignoring --setgroup ceph since I am not root 2024-04-10T03:02:29.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-mgr[35386]: -- 172.21.15.152:0/2677656204 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563c386f69c0 con 0x563c386cd000 2024-04-10T03:02:30.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:29.954+0000 7fc4658a31c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:02:30.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:29.936+0000 7f78b88a21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:02:30.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:30.025+0000 7f78b88a21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:02:30.350 INFO:teuthology.orchestra.run.smithi102.stdout:[client.0] 2024-04-10T03:02:30.350 INFO:teuthology.orchestra.run.smithi102.stdout: key = AQBGARZmcC7SFBAAQVdmWNMAQ7ZOsDliAktwLQ== 2024-04-10T03:02:30.351 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:30.049+0000 7fc4658a31c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:02:30.654 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:30.654 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:30.654 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:30.654 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:30.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:30.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4273135960' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:30.815 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:30.646+0000 7f78b88a21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:02:30.815 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:30.734+0000 7f78b88a21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:02:31.006 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:30.685+0000 7fc4658a31c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:02:31.006 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:30.774+0000 7fc4658a31c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:02:31.006 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:30.858+0000 7fc4658a31c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:02:31.006 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.007+0000 7fc4658a31c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:02:31.072 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T03:02:31.072 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-10T03:02:31.072 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-10T03:02:31.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-10T03:02:31.142 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:30.816+0000 7f78b88a21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:02:31.142 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:30.965+0000 7f78b88a21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:02:31.142 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.043+0000 7f78b88a21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:02:31.263 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.092+0000 7fc4658a31c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:02:31.385 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.b/config 2024-04-10T03:02:31.418 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.217+0000 7f78b88a21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:02:31.418 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.419+0000 7f78b88a21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:02:31.524 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.264+0000 7fc4658a31c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:02:31.524 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.442+0000 7fc4658a31c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:02:31.524 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.525+0000 7fc4658a31c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:02:31.686 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.504+0000 7f78b88a21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:02:31.686 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.599+0000 7f78b88a21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:02:31.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.618+0000 7fc4658a31c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:02:31.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:31.702+0000 7fc4658a31c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:02:32.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:31.687+0000 7f78b88a21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:02:32.777 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:32.741+0000 7f78b88a21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:02:33.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:32.734+0000 7fc4658a31c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:02:33.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:32.861+0000 7fc4658a31c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:02:33.072 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:32.873+0000 7f78b88a21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:02:33.072 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:33.073+0000 7f78b88a21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:02:33.308 INFO:teuthology.orchestra.run.smithi152.stdout:[client.1] 2024-04-10T03:02:33.309 INFO:teuthology.orchestra.run.smithi152.stdout: key = AQBJARZmPMdCEhAAB4L390xptCJcRGlzEy92EA== 2024-04-10T03:02:33.319 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:33.053+0000 7fc4658a31c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:02:33.319 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:33.291+0000 7fc4658a31c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:02:33.322 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:33.310+0000 7f78b88a21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:02:33.663 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:33.478+0000 7fc4658a31c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[29301]: from='client.? 172.21.15.152:0/3895067723' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[29301]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[29301]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[39651]: from='client.? 172.21.15.152:0/3895067723' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.664 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-mon[33523]: from='client.? 172.21.15.152:0/3895067723' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-10T03:02:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-10T03:02:33.686 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:33.498+0000 7f78b88a21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:02:33.686 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:33.687+0000 7f78b88a21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:02:33.930 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:33.665+0000 7fc4658a31c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:02:33.993 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-10T03:02:33.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-10T03:02:33.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-10T03:02:34.078 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-10T03:02:34.078 DEBUG:teuthology.orchestra.run.smithi102:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-10T03:02:34.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:33.955+0000 7f78b88a21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:02:34.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:34.046+0000 7f78b88a21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:02:34.285 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:33.932+0000 7fc4658a31c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:02:34.285 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:34.019+0000 7fc4658a31c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:02:34.369 INFO:teuthology.misc.health.smithi102.stdout: 2024-04-10T03:02:34.369 INFO:teuthology.misc.health.smithi102.stdout:{"epoch":58,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:02:08.300519+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:02:34.380 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-10T03:02:34.380 INFO:tasks.cephadm:Creating RBD pool 2024-04-10T03:02:34.381 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-10T03:02:34.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:34 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4110525393' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:34.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:34 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4110525393' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:34.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:34.422+0000 7fc4658a31c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:02:34.642 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:34.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:34 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4110525393' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:34.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:34.447+0000 7f78b88a21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:02:35.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:35 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:35.243+0000 7fc4658a31c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:02:35.496 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:35.210+0000 7f78b88a21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:02:35.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:35.497+0000 7f78b88a21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:02:35.858 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:35 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:35.544+0000 7fc4658a31c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: Standby manager daemon x restarted 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:02:36.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:35.811+0000 7f78b88a21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:36.001+0000 7f78b88a21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: [10/Apr/2024:03:02:36] ENGINE Bus STARTING 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: CherryPy Checker: 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: The Application mounted at '' has an empty config. 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: [10/Apr/2024:03:02:36] ENGINE Serving on http://:::9283 2024-04-10T03:02:36.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: [10/Apr/2024:03:02:36] ENGINE Bus STARTED 2024-04-10T03:02:36.125 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:35 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:35.859+0000 7fc4658a31c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:02:36.125 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:36.063+0000 7fc4658a31c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:02:36.125 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: Standby manager daemon x restarted 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: Standby manager daemon x restarted 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: Standby manager daemon x started 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:02:36.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/991519765' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:36] ENGINE Bus STARTING 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: CherryPy Checker: 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: The Application mounted at '' has an empty config. 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:36] ENGINE Serving on http://:::9283 2024-04-10T03:02:36.429 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:36 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:36] ENGINE Bus STARTED 2024-04-10T03:02:37.129 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'rbd' created 2024-04-10T03:02:37.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-10T03:02:37.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: Active manager daemon y restarted 2024-04-10T03:02:37.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: Activating manager daemon y 2024-04-10T03:02:37.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: osdmap e59: 8 total, 8 up, 8 in 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: mgrmap e18: y(active, starting, since 0.00621373s), standbys: x 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:02:37.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: Manager daemon y is now available 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-10T03:02:37.231 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: Active manager daemon y restarted 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: Activating manager daemon y 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: osdmap e59: 8 total, 8 up, 8 in 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: mgrmap e18: y(active, starting, since 0.00621373s), standbys: x 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:02:37.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: Manager daemon y is now available 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:37.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: Active manager daemon y restarted 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: Activating manager daemon y 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: osdmap e59: 8 total, 8 up, 8 in 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: mgrmap e18: y(active, starting, since 0.00621373s), standbys: x 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:02:37.295 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:02:37.296 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: Manager daemon y is now available 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-10T03:02:37.297 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:38.130 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: [10/Apr/2024:03:02:36] ENGINE Bus STARTING 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: [10/Apr/2024:03:02:36] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: [10/Apr/2024:03:02:36] ENGINE Bus STARTED 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: mgrmap e19: y(active, since 1.01198s), standbys: x 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:38.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-10T03:02:38.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:38 smithi152 ceph-mon[33523]: osdmap e60: 8 total, 8 up, 8 in 2024-04-10T03:02:38.407 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:38.529 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: [10/Apr/2024:03:02:36] ENGINE Bus STARTING 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: [10/Apr/2024:03:02:36] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: [10/Apr/2024:03:02:36] ENGINE Bus STARTED 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: mgrmap e19: y(active, since 1.01198s), standbys: x 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[29301]: osdmap e60: 8 total, 8 up, 8 in 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: [10/Apr/2024:03:02:36] ENGINE Bus STARTING 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: [10/Apr/2024:03:02:36] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: [10/Apr/2024:03:02:36] ENGINE Bus STARTED 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: mgrmap e19: y(active, since 1.01198s), standbys: x 2024-04-10T03:02:38.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:38.531 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/380130188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-10T03:02:38.531 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:38 smithi102 ceph-mon[39651]: osdmap e60: 8 total, 8 up, 8 in 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:02:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: osdmap e61: 8 total, 8 up, 8 in 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:39 smithi152 ceph-mon[33523]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: osdmap e61: 8 total, 8 up, 8 in 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[29301]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: osdmap e61: 8 total, 8 up, 8 in 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:39.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:39 smithi102 ceph-mon[39651]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:02:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: osdmap e62: 8 total, 8 up, 8 in 2024-04-10T03:02:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: osdmap e62: 8 total, 8 up, 8 in 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:02:40.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:40 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: osdmap e62: 8 total, 8 up, 8 in 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:02:40.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 4016M 2024-04-10T03:02:40.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:40.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:02:40.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:02:40.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:02:40.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:40 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:02:41.304 INFO:teuthology.orchestra.run.smithi102.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:41.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: Deploying daemon alertmanager.a on smithi102 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3402772529' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:41.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:41 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: Deploying daemon alertmanager.a on smithi102 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3402772529' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:02:41.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:02:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:02:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: mgrmap e21: y(active, since 4s), standbys: x 2024-04-10T03:02:41.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: Deploying daemon alertmanager.a on smithi102 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3402772529' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:41.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:41 smithi102 ceph-mon[29301]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:02:42.629 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:42 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:02:42.630 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:42 smithi102 ceph-mon[39651]: osdmap e63: 8 total, 8 up, 8 in 2024-04-10T03:02:42.630 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:42 smithi102 ceph-mon[29301]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:02:42.630 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:42 smithi102 ceph-mon[29301]: osdmap e63: 8 total, 8 up, 8 in 2024-04-10T03:02:42.653 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-10T03:02:42.654 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-10T03:02:42.654 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-10T03:02:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:42 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:02:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:42 smithi152 ceph-mon[33523]: osdmap e63: 8 total, 8 up, 8 in 2024-04-10T03:02:42.912 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:43.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:43 smithi152 ceph-mon[33523]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:43.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:43 smithi152 ceph-mon[33523]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:02:43.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:43 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[29301]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[29301]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[29301]: Cluster is now healthy 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[39651]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[39651]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:02:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:43 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:02:45.196 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:02:45.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:45 smithi102 ceph-mon[29301]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-10T03:02:45.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:45 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3367265147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:02:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:45 smithi102 ceph-mon[39651]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-10T03:02:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:45 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3367265147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:02:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:45 smithi152 ceph-mon[33523]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-10T03:02:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:45 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3367265147' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:02:46.342 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":21,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":3562317133},{"type":"v1","addr":"172.21.15.102:6801","nonce":3562317133}]},"active_addr":"172.21.15.102:6801/3562317133","active_change":"2024-04-10T03:02:36.066020+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.102:8443/","prometheus":"http://172.21.15.102:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1928572397}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":2965564602}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":2292901015}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1827401448}]}]} 2024-04-10T03:02:46.345 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-10T03:02:46.345 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-10T03:02:46.346 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:02:47.620 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:47 smithi102 ceph-mon[39651]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-10T03:02:47.620 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:47 smithi102 ceph-mon[29301]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-10T03:02:47.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:47 smithi152 ceph-mon[33523]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-10T03:02:48.515 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:49.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:49 smithi152 ceph-mon[33523]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-10T03:02:49.513 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:02:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:02:49.608 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:49 smithi102 ceph-mon[29301]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-10T03:02:49.608 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:49 smithi102 ceph-mon[39651]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-10T03:02:50.261 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:50 smithi102 systemd[1]: Starting Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:02:50.978 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:50 smithi102 podman[68363]: 2024-04-10 03:02:50.586590854 +0000 UTC m=+0.018366297 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-10T03:02:50.978 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:50 smithi102 podman[68363]: 2024-04-10 03:02:50.686180725 +0000 UTC m=+0.117956166 volume create 80d5b048063f2bc2b8a65e14631d7cd20868289612e67ab6328992dab5a5e81d 2024-04-10T03:02:50.978 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:50 smithi102 podman[68363]: 2024-04-10 03:02:50.769987288 +0000 UTC m=+0.201762786 container create 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.339 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.340 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 podman[68363]: 2024-04-10 03:02:51.089236743 +0000 UTC m=+0.521012179 container init 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 podman[68363]: 2024-04-10 03:02:51.091834645 +0000 UTC m=+0.523610084 container start 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.103Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.103Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.104Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.102 port=9094 2024-04-10T03:02:51.340 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.105Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.128Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.128Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.131Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:51.131Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 bash[68363]: 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff 2024-04-10T03:02:51.341 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:51 smithi102 systemd[1]: Started Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:02:51.605 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE Bus STOPPING 2024-04-10T03:02:51.605 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:02:51.605 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE Bus STOPPED 2024-04-10T03:02:51.605 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE Bus STARTING 2024-04-10T03:02:51.606 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE Serving on http://:::9283 2024-04-10T03:02:51.606 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:02:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:02:51] ENGINE Bus STARTED 2024-04-10T03:02:51.648 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:02:51.649 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":63,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:02:41.294816+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.308 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-10T03:02:52.309 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: Deploying daemon grafana.a on smithi152 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2706446401' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[29301]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: Deploying daemon grafana.a on smithi152 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2706446401' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:52.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:52 smithi102 ceph-mon[39651]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-10T03:02:52.563 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: Deploying daemon grafana.a on smithi152 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2706446401' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:52.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:52 smithi152 ceph-mon[33523]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-10T03:02:53.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:02:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:02:53.105Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000417275s 2024-04-10T03:02:54.348 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:02:54.349 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":63,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:02:41.294816+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:02:54.974 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-10T03:02:54.974 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-10T03:02:54.975 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-10T03:02:54.975 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-10T03:02:54.975 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-10T03:02:54.976 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-10T03:02:54.976 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-10T03:02:54.977 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-10T03:02:55.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:55 smithi102 ceph-mon[39651]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-10T03:02:55.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:55 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1738096499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:55.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:55 smithi102 ceph-mon[29301]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-10T03:02:55.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:55 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1738096499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:55 smithi152 ceph-mon[33523]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-10T03:02:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:55 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1738096499' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:02:55.542 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.542 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.543 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.543 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.544 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.548 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.548 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:55.548 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:02:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:57 smithi102 ceph-mon[29301]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:57 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:57.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:57 smithi102 ceph-mon[39651]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:57.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:57 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:57.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:57 smithi152 ceph-mon[33523]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:57.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:57 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:02:59.469 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:02:59 smithi102 ceph-mon[29301]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:59.469 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:02:59 smithi102 ceph-mon[39651]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:59.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:02:59 smithi152 ceph-mon[33523]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:02:59.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:02:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:02:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:01.389 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:01 smithi102 ceph-mon[29301]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:01.390 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:01 smithi102 ceph-mon[39651]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:01.390 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:03:01.107Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001627343s 2024-04-10T03:03:01.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:01 smithi152 ceph-mon[33523]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:03.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:03 smithi102 ceph-mon[39651]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:03.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:03 smithi102 ceph-mon[29301]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:03.348 INFO:teuthology.orchestra.run.smithi102.stdout:34359738415 2024-04-10T03:03:03.349 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-10T03:03:03.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:03 smithi152 ceph-mon[33523]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:04.888 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:04 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:05.191 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-mon[33523]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:05.191 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 podman[62851]: 2024-04-10 03:03:04.960670388 +0000 UTC m=+0.017526366 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:03:05.191 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 podman[62851]: 2024-04-10 03:03:05.081362753 +0000 UTC m=+0.138218709 container create 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, version=8.5, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vcs-type=git, distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-04-10T03:03:05.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:05 smithi102 ceph-mon[29301]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:05.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:05 smithi102 ceph-mon[39651]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:05.481 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 podman[62851]: 2024-04-10 03:03:05.317402161 +0000 UTC m=+0.374258145 container init 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.expose-services=, name=ubi8, vcs-type=git, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-10T03:03:05.481 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 podman[62851]: 2024-04-10 03:03:05.320294452 +0000 UTC m=+0.377150412 container start 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, distribution-scope=public, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.openshift.tags=base rhel8, version=8.5, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:03:05.481 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 bash[62851]: 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae 2024-04-10T03:03:05.481 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483726082Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483904205Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483929615Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483941396Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483951926Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-10T03:03:05.733 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483962355Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483974896Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483985583Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.483996441Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.484005109Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.484012361Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.484018084Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.484024073Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=settings t=2024-04-10T03:03:05.4840305Z level=info msg="App mode production" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore t=2024-04-10T03:03:05.484095102Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore t=2024-04-10T03:03:05.484114103Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.484979085Z level=info msg="Starting DB migrations" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.485061458Z level=info msg="Executing migration" id="create migration_log table" 2024-04-10T03:03:05.734 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.485567832Z level=info msg="Executing migration" id="create user table" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.486002543Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.486758046Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.487548016Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.490372102Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.493193952Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-10T03:03:05.735 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.496007152Z level=info msg="Executing migration" id="create user table v2" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.498701204Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.501425575Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.504264562Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.506989059Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.509765078Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.512588678Z level=info msg="Executing migration" id="Update user table charset" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.515332896Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.515799823Z level=info msg="Executing migration" id="Add missing user data" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.518552643Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.521222526Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.523896628Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.526548115Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.529142305Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.531818802Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-10T03:03:05.736 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.53452101Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.537025712Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.539631966Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.542214109Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.544872466Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.54752259Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.549990443Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.550383587Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.550753755Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.551439007Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.551847834Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.552257973Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.552662924Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.553075225Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.553486689Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-10T03:03:05.737 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.553798416Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.554160189Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.554511102Z level=info msg="Executing migration" id="create star table" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.554868051Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.555301604Z level=info msg="Executing migration" id="create org table v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.555689945Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.556102556Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.556483777Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.556903928Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.557346005Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.55773253Z level=info msg="Executing migration" id="Update org table charset" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.558001169Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.558300508Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.558591054Z level=info msg="Executing migration" id="create dashboard table" 2024-04-10T03:03:05.738 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.558970958Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.559364741Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.559761427Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.560228963Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.561070876Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.561488863Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.562340249Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.563186469Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-10T03:03:05.739 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.5640401Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.564474086Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.564802074Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.56530664Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.566170734Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.566677507Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.567182466Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-10T03:03:05.740 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.56807198Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.568959434Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.569853921Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.570322184Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.571144614Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.571989917Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.572291328Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.57280099Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.573639432Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.574516728Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.575388501Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.575732207Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.576646052Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.577485498Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.578334573Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-10T03:03:05.741 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.578779455Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.579692336Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.580628608Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.581473468Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.58190748Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.584804133Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.587704819Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.590551038Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.591128431Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.593982809Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.596709153Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.599442526Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.599905295Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.602833708Z level=info msg="Executing migration" id="create data_source table" 2024-04-10T03:03:05.742 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.605753839Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.60623006Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.608600668Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.611495752Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.614488671Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.61814264Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.618582077Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.621531977Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.622001251Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.624307928Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.627025637Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.627590318Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.630428853Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.630778941Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-10T03:03:05.743 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.633167258Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.635905282Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.636258151Z level=info msg="Executing migration" id="Add uid column" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.639018537Z level=info msg="Executing migration" id="Update uid value" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.639372594Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.641655103Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.642117308Z level=info msg="Executing migration" id="create api_key table" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.644459598Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.644916662Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.647660876Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.650211289Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.653000501Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.653841358Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.654703909Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.65578188Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-10T03:03:05.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.656651341Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.657496273Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.658361489Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.659188235Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.659559439Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.660003021Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.660318672Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.660848743Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.661426711Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.661790582Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.666814516Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.667436158Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.668280752Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.669165339Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.670050164Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.67090495Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-10T03:03:05.745 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.671793566Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.672665357Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.673538225Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.674410026Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.675273849Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.676156112Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.677043158Z level=info msg="Executing migration" id="create quota table v1" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.677891002Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.67874268Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.679605359Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.680065858Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.680918669Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.681772704Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.682597292Z level=info msg="Executing migration" id="create session table" 2024-04-10T03:03:05.746 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.683050517Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.683855787Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.684198981Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.684621669Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.685456712Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.686303331Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.686645575Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.686969202Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.687310544Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.687730775Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.688035505Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.688667347Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.688985923Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.691511031Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.694391719Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.69475344Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-10T03:03:05.747 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.697307384Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-10T03:03:05.749 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.698498693Z level=info msg="Executing migration" id="create alert table v1" 2024-04-10T03:03:05.749 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.700292383Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-10T03:03:05.749 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.702113356Z level=info msg="Executing migration" id="add index alert state" 2024-04-10T03:03:05.749 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.70396662Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-10T03:03:05.749 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.705755461Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.707598527Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.709399344Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.711270846Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.71314718Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.714972084Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.716795179Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.718612423Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.720456078Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.722279618Z level=info msg="Executing migration" id="Add column is_default" 2024-04-10T03:03:05.750 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.724159408Z level=info msg="Executing migration" id="Add column frequency" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.725983033Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.727838169Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.72968685Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.731536301Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.733352719Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.735184756Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.737042366Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.73887876Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.740716327Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-10T03:03:05.751 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.742549735Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-10T03:03:05.983 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.74436778Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-10T03:03:05.983 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.746186448Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-10T03:03:05.983 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.748025189Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-10T03:03:05.983 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.749845561Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-10T03:03:05.983 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.75167319Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.753539731Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.755336861Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.757166612Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.759027292Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.760910241Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.762799401Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.764692919Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.766571422Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.768449189Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.77034615Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.772208423Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.774102229Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.77601719Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-10T03:03:05.984 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.777944649Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.779818221Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.78170161Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.783588141Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.785472268Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.787375336Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.789241655Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.79112713Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.793033467Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.794943035Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.796841177Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.798719433Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.800623153Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.802511292Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.804400748Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-10T03:03:05.985 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.806258898Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-10T03:03:05.986 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.808156946Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-10T03:03:05.986 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.810083444Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-10T03:03:05.986 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.81199727Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-10T03:03:05.986 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.813875695Z level=info msg="Executing migration" id="Move region to single row" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.815758169Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.817653274Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.819560561Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.821437243Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.823341841Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.825193932Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.82912083Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.833000591Z level=info msg="Executing migration" id="create test_data table" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.837012018Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.840811064Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.844719762Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-10T03:03:05.987 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.848619223Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.852591358Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.856379816Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.860393043Z level=info msg="Executing migration" id="create team table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.864227229Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.868163829Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.872050179Z level=info msg="Executing migration" id="create team member table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.875872986Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.879774703Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.883566536Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.887376864Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.891170776Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.895160144Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.89710097Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.898875324Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-10T03:03:05.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.900774851Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.90253823Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.904497918Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.906268708Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.908231097Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.908814323Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.910861652Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.912748989Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.914610384Z level=info msg="Executing migration" id="create tag table" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.915022797Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.916937589Z level=info msg="Executing migration" id="create login attempt table" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.918909838Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.920715833Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.922651306Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-10T03:03:05.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.92482807Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.926750666Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.928747494Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.929182427Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.931159378Z level=info msg="Executing migration" id="create user auth table" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.932978924Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.93501958Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.93538451Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.937306585Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.939173911Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.941117619Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.941880848Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.943921308Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.945769126Z level=info msg="Executing migration" id="create server_lock table" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.947567888Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-10T03:03:05.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.948036913Z level=info msg="Executing migration" id="create user auth token table" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.950076269Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.951811976Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.955860153Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.956341572Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.960560672Z level=info msg="Executing migration" id="create cache_data table" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.96447796Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.968714225Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.969288942Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.972178788Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.974764937Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.975137581Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.975610921Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.978246412Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-10T03:03:05.991 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.978698727Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.978975231Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.979394989Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.979775481Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.980265968Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.980688781Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.981538717Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.981978152Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.982285018Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.982662032Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.983090699Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.98350009Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.98376492Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.984196674Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-10T03:03:05.992 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.984602914Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-10T03:03:05.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.984993257Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-10T03:03:05.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.985418703Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-10T03:03:05.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.986546983Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-10T03:03:05.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.987123431Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-10T03:03:05.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.987585729Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-10T03:03:06.200 INFO:teuthology.orchestra.run.smithi102.stdout:77309411366 2024-04-10T03:03:06.201 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-10T03:03:06.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.990954904Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-10T03:03:06.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.99428783Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.99472595Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.99515475Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.996000035Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.996565579Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.997187978Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.997783446Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.9983651Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:05.998768642Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.000002083Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.001226099Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.002478562Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.00304116Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.003632675Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-10T03:03:06.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.004873453Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.006118289Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.00676046Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.007891555Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.0087483Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.00905828Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.00953182Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.012194927Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.012713739Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.013001323Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.013852137Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.015944732Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.018414339Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-10T03:03:06.264 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.021174229Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.022043338Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.024943922Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.027654769Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.03034907Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.030791223Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.033603075Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-10T03:03:06.265 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.034456726Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-10T03:03:06.266 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.035341809Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-10T03:03:06.266 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.035739106Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-10T03:03:06.266 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.036766175Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-10T03:03:06.266 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.037773264Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-10T03:03:06.266 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.038619708Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.039087863Z level=info msg="Executing migration" id="create alert_image table" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.039949106Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.040774077Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.041618558Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.042018032Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.042453467Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.042528697Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.043248495Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.043617796Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.044094824Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.044210665Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.044317499Z level=info msg="alerts found to migrate" alerts=0 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.044710553Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.045561045Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-10T03:03:06.267 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.046585533Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.047389111Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.048160082Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.048652383Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.049573004Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.050228063Z level=info msg="Executing migration" id="create data_keys table" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.050671505Z level=info msg="Executing migration" id="create secrets table" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.051566989Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.055893533Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.056833788Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.057817079Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.062150936Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.066387806Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.067356906Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.068076984Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-10T03:03:06.268 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.068914736Z level=info msg="Executing migration" id="create permission table" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.069324305Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.070174309Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.071028165Z level=info msg="Executing migration" id="create role table" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.073837475Z level=info msg="Executing migration" id="add column display_name" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.074781475Z level=info msg="Executing migration" id="add column group_name" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.07777662Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.080772126Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.083509627Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.084051021Z level=info msg="Executing migration" id="create team role table" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.086970223Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.089660186Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.092482874Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.09293924Z level=info msg="Executing migration" id="create user role table" 2024-04-10T03:03:06.269 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.096170061Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.099161295Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.101977232Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.102435556Z level=info msg="Executing migration" id="create builtin role table" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.105372345Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.108379193Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.111266479Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.112264049Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.112756818Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.115332052Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.116004032Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.116499885Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.119242345Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.119742011Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-10T03:03:06.270 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.120731593Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.121159343Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.12165567Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.12193756Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.122400802Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.122848927Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.123241973Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.123558178Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.123909755Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.124355453Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.124807468Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.125787143Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.12614925Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.126591577Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.127089812Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-10T03:03:06.271 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.129811935Z level=info msg="Executing migration" id="add correlation config column" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.130834905Z level=info msg="Executing migration" id="create entity_events table" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.131298919Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.131743465Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.131847569Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.134508583Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.134604581Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.137181407Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.14004507Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.140517849Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.143300072Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.145859224Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.148635479Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.149090337Z level=info msg="Executing migration" id="Drop public config table" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.151973187Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-10T03:03:06.272 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.154636163Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.157200841Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.159911058Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.162603204Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.165599782Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.166592588Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.167667949Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.168556629Z level=info msg="Executing migration" id="add share column" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.169605023Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.170612882Z level=info msg="Executing migration" id="create file table" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.17118012Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.172127349Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.172892222Z level=info msg="Executing migration" id="create file_meta table" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.173752561Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.174226915Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-10T03:03:06.273 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.175152936Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.176023154Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.176642054Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.17727214Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.178291675Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.179166304Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.180060344Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.180752149Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.181708403Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.182447496Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.18349723Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.184528549Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.185385795Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.195249781Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-10T03:03:06.274 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.196233784Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.196721285Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.200199369Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.201135618Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.201951321Z level=info msg="Executing migration" id="create folder table" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.202371613Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.203326341Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.204078409Z level=info msg="Executing migration" id="Update folder title length" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.204899498Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=migrator t=2024-04-10T03:03:06.205364489Z level=info msg="migrations completed" performed=485 skipped=0 duration=720.32857ms 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore t=2024-04-10T03:03:06.205761517Z level=info msg="Created default organization" 2024-04-10T03:03:06.275 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=plugin.loader t=2024-04-10T03:03:06.264608844Z level=info msg="Plugin registered" pluginID=input 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: Deploying daemon node-exporter.a on smithi102 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:06.763 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=plugin.loader t=2024-04-10T03:03:06.296569596Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-10T03:03:06.763 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=plugin.loader t=2024-04-10T03:03:06.296589206Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-10T03:03:06.763 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=secrets t=2024-04-10T03:03:06.296622118Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-10T03:03:06.763 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=query_data t=2024-04-10T03:03:06.298315374Z level=info msg="Query Service initialization" 2024-04-10T03:03:06.763 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=live.push_http t=2024-04-10T03:03:06.300535853Z level=info msg="Live Push Gateway initialization" 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: Deploying daemon node-exporter.a on smithi102 2024-04-10T03:03:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: Deploying daemon node-exporter.a on smithi102 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:06.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=infra.usagestats.collector t=2024-04-10T03:03:07.032035744Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=provisioning.datasources t=2024-04-10T03:03:07.032327136Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=provisioning.datasources t=2024-04-10T03:03:07.064993946Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=provisioning.datasources t=2024-04-10T03:03:07.097633638Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=provisioning.alerting t=2024-04-10T03:03:07.119405569Z level=info msg="starting to provision alerting" 2024-04-10T03:03:07.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=provisioning.alerting t=2024-04-10T03:03:07.119424667Z level=info msg="finished to provision alerting" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=ngalert.state.manager t=2024-04-10T03:03:07.119495067Z level=info msg="Warming state cache for startup" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=grafanaStorageLogger t=2024-04-10T03:03:07.119675911Z level=info msg="storage starting" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=http.server t=2024-04-10T03:03:07.123969327Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-10T03:03:07.129899711Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=ngalert.state.manager t=2024-04-10T03:03:07.129918941Z level=info msg="State cache has been initialized" states=0 duration=10.421605ms 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=ngalert.multiorg.alertmanager t=2024-04-10T03:03:07.129970502Z level=info msg="starting MultiOrg Alertmanager" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=ticker t=2024-04-10T03:03:07.13000372Z level=info msg=starting first_tick=2024-04-10T03:03:10Z 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-10T03:03:07.176510866Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-10T03:03:07.206795186Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-10T03:03:07.513 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:03:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-10T03:03:07.217137536Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-10T03:03:09.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:09 smithi152 ceph-mon[33523]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:09.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:03:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:03:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:09.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:09 smithi102 ceph-mon[39651]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:09.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:09 smithi102 ceph-mon[29301]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:10.639 INFO:teuthology.orchestra.run.smithi102.stdout:128849018909 2024-04-10T03:03:10.639 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-10T03:03:11.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:11 smithi152 ceph-mon[33523]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:11.540 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:11 smithi102 ceph-mon[29301]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:11.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:11 smithi102 ceph-mon[39651]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:11.842 INFO:teuthology.orchestra.run.smithi102.stdout:197568495636 2024-04-10T03:03:11.842 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-10T03:03:12.526 INFO:teuthology.orchestra.run.smithi102.stdout:55834574892 2024-04-10T03:03:12.526 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-10T03:03:13.042 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:12 smithi102 bash[70204]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-10T03:03:13.383 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:13 smithi102 ceph-mon[29301]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:13.383 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:13 smithi102 ceph-mon[39651]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:13.384 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:13 smithi102 bash[70204]: Getting image source signatures 2024-04-10T03:03:13.384 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:13 smithi102 bash[70204]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-10T03:03:13.384 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:13 smithi102 bash[70204]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-10T03:03:13.384 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:13 smithi102 bash[70204]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-10T03:03:13.409 INFO:teuthology.orchestra.run.smithi102.stdout:163208757271 2024-04-10T03:03:13.409 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-10T03:03:13.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:13 smithi152 ceph-mon[33523]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:14.028 INFO:teuthology.orchestra.run.smithi102.stdout:231928233999 2024-04-10T03:03:14.028 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-10T03:03:14.396 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:15.139 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:15 smithi102 bash[70204]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-10T03:03:15.447 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:15 smithi102 ceph-mon[29301]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:15.447 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:15 smithi102 ceph-mon[39651]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:15.447 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:15 smithi102 bash[70204]: Writing manifest to image destination 2024-04-10T03:03:15.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:15 smithi152 ceph-mon[33523]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:15.619 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:15.621 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:15.621 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:15.621 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:15.665 INFO:teuthology.orchestra.run.smithi102.stdout:107374182435 2024-04-10T03:03:15.665 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-10T03:03:15.775 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:15 smithi102 podman[70204]: 2024-04-10 03:03:15.646932259 +0000 UTC m=+3.540447935 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-10T03:03:16.042 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:15 smithi102 podman[70204]: 2024-04-10 03:03:15.996375524 +0000 UTC m=+3.889891186 container create 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 podman[70204]: 2024-04-10 03:03:16.862014974 +0000 UTC m=+4.755530639 container init 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 podman[70204]: 2024-04-10 03:03:16.864953229 +0000 UTC m=+4.758468894 container start 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.870Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.870Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=arp 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-10T03:03:17.043 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=edac 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-10T03:03:17.044 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=os 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-10T03:03:17.045 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=stat 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=time 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=uname 2024-04-10T03:03:17.046 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-10T03:03:17.047 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-10T03:03:17.047 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-10T03:03:17.047 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-10T03:03:17.047 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[70633]: ts=2024-04-10T03:03:16.871Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-10T03:03:17.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:17 smithi152 ceph-mon[33523]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:17 smithi102 ceph-mon[29301]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:17.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:17 smithi102 ceph-mon[39651]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:17.995 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:17 smithi102 bash[70204]: 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 2024-04-10T03:03:17.995 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:03:17 smithi102 systemd[1]: Started Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:19.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:18 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:18 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:18 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:18 smithi152 ceph-mon[33523]: Deploying daemon node-exporter.b on smithi152 2024-04-10T03:03:19.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:18 smithi152 ceph-mon[33523]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:19.157 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 systemd[1]: Starting Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[29301]: Deploying daemon node-exporter.b on smithi152 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[29301]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:19.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:19.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[39651]: Deploying daemon node-exporter.b on smithi152 2024-04-10T03:03:19.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:18 smithi102 ceph-mon[39651]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:19.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:03:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:03:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:19.512 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 bash[63139]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-10T03:03:20.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 bash[63139]: Getting image source signatures 2024-04-10T03:03:20.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 bash[63139]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-10T03:03:20.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 bash[63139]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-10T03:03:20.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:19 smithi152 bash[63139]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-10T03:03:20.610 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:20.611 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:21.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-mon[33523]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:21.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2018351513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:03:21.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:21.262 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:20 smithi152 bash[63139]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-10T03:03:21.262 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 bash[63139]: Writing manifest to image destination 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[29301]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2018351513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[39651]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2018351513' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:03:21.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:21 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 podman[63139]: 2024-04-10 03:03:21.341108651 +0000 UTC m=+2.163910612 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 podman[63139]: 2024-04-10 03:03:21.431333788 +0000 UTC m=+2.254135791 container create 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 podman[63139]: 2024-04-10 03:03:21.617054614 +0000 UTC m=+2.439856572 container init 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 podman[63139]: 2024-04-10 03:03:21.619869858 +0000 UTC m=+2.442671837 container start 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.624Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.624Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.624Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.624Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.624Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-10T03:03:21.718 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=arp 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=edac 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-10T03:03:21.719 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=os 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-10T03:03:21.720 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=stat 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=time 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=uname 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-10T03:03:21.721 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.625Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-10T03:03:21.722 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.626Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-10T03:03:21.722 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[63190]: ts=2024-04-10T03:03:21.626Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-10T03:03:21.722 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 bash[63139]: 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 2024-04-10T03:03:22.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:03:21 smithi152 systemd[1]: Started Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:22.488 INFO:teuthology.orchestra.run.smithi102.stdout:77309411369 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:23.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3334651353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:03:23.025 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.025 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3334651353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.026 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.027 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.027 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:23.027 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:23.027 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:23.027 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3334651353' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:03:23.331 INFO:teuthology.orchestra.run.smithi102.stdout:34359738420 2024-04-10T03:03:24.132 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:23 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1831521545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:03:24.132 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:23 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1831521545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:03:24.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:23 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1831521545' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:03:24.459 INFO:teuthology.orchestra.run.smithi102.stdout:55834574894 2024-04-10T03:03:24.979 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:24 smithi102 ceph-mon[39651]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:24.979 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:24 smithi102 ceph-mon[29301]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:25.013 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574894 for osd.1 2024-04-10T03:03:25.013 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:25.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:24 smithi152 ceph-mon[33523]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:26.041 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738420 for osd.0 2024-04-10T03:03:26.041 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:26.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.693 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.694 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.694 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:26 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:26 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:26 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:26.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:26 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:27.352 INFO:teuthology.orchestra.run.smithi102.stdout:197568495639 2024-04-10T03:03:27.700 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:27 smithi102 ceph-mon[29301]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:27.700 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:27 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1463132554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:03:27.701 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:27 smithi102 ceph-mon[39651]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:27.701 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:27 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1463132554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:03:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:27 smithi152 ceph-mon[33523]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:27 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1463132554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:03:27.797 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:27.998 INFO:teuthology.orchestra.run.smithi102.stdout:231928234001 2024-04-10T03:03:28.223 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495636 got 197568495639 for osd.6 2024-04-10T03:03:28.224 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:28.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:28 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/854241522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:03:28.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:28 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/854241522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:03:28.631 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411369 for osd.2 2024-04-10T03:03:28.631 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:28 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/854241522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:03:29.153 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234001 for osd.7 2024-04-10T03:03:29.153 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:29.433 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:03:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:03:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:29.541 INFO:teuthology.orchestra.run.smithi102.stdout:163208757275 2024-04-10T03:03:29.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:29 smithi102 ceph-mon[29301]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:29 smithi102 ceph-mon[39651]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:29.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:29 smithi152 ceph-mon[33523]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:30.199 INFO:teuthology.orchestra.run.smithi102.stdout:128849018913 2024-04-10T03:03:30.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2577087308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:03:30.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1442150436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:03:30.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2577087308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:03:30.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1442150436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:03:30.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2577087308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:03:30.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1442150436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:03:30.948 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018909 got 128849018913 for osd.4 2024-04-10T03:03:30.948 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:31.701 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:31 smithi102 ceph-mon[29301]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:31.701 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:31 smithi102 ceph-mon[39651]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:31.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:31 smithi152 ceph-mon[33523]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:32.415 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757271 got 163208757275 for osd.5 2024-04-10T03:03:32.415 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:33.030 INFO:teuthology.orchestra.run.smithi102.stdout:107374182439 2024-04-10T03:03:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:33 smithi152 ceph-mon[33523]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:33 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3769998648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:03:33.773 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:33 smithi102 ceph-mon[29301]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:33.773 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:33 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3769998648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:03:33.773 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:33 smithi102 ceph-mon[39651]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:33.773 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:33 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3769998648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:03:33.796 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182435 got 107374182439 for osd.3 2024-04-10T03:03:33.797 DEBUG:teuthology.parallel:result is None 2024-04-10T03:03:33.797 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-10T03:03:33.797 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:03:35.324 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:35 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:35.776 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:35.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:35 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:36.740 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[39651]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:03:36.740 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[39651]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:03:36.741 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:36.741 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[29301]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:03:36.741 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[29301]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:03:36.741 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:36 smithi152 ceph-mon[33523]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:03:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:36 smithi152 ceph-mon[33523]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:03:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:37 smithi152 ceph-mon[33523]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:37 smithi102 ceph-mon[29301]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:37.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:37 smithi102 ceph-mon[39651]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:39.461 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:03:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:03:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:39.493 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:39 smithi102 ceph-mon[29301]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:39.493 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:39 smithi102 ceph-mon[39651]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:39.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:39 smithi152 ceph-mon[33523]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:39.792 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:39 smithi102 systemd[1]: Stopping Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:40.039 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:03:40.040 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:03:40.073 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:39 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[68379]: ts=2024-04-10T03:03:39.976Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:03:40.074 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:39 smithi102 podman[72225]: 2024-04-10 03:03:39.99646795 +0000 UTC m=+0.143998751 container died 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:40.633 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:40 smithi102 podman[72225]: 2024-04-10 03:03:40.4179463 +0000 UTC m=+0.565477093 container cleanup 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:40.633 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:40 smithi102 bash[72225]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a 2024-04-10T03:03:41.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:40 smithi102 podman[72239]: 2024-04-10 03:03:40.635825398 +0000 UTC m=+0.640311153 container remove 7e26f4bed3bd49d01a2fc7536f8626d2f1caaf7695a778af66c6b4260bdca7ff (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:41.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:40 smithi102 podman[72239]: 2024-04-10 03:03:40.711311378 +0000 UTC m=+0.715797084 volume remove 80d5b048063f2bc2b8a65e14631d7cd20868289612e67ab6328992dab5a5e81d 2024-04-10T03:03:41.416 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-10T03:03:38.086980+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385500,"kb_used_data":6652,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575716,"statfs":{"total":767960285184,"available":765517533184,"internally_reserved":0,"allocated":6811648,"data_stored":3875712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001742"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479682+0000","last_change":"2024-04-10T03:02:38.143240+0000","last_active":"2024-04-10T03:02:39.479682+0000","last_peered":"2024-04-10T03:02:39.479682+0000","last_clean":"2024-04-10T03:02:39.479682+0000","last_became_active":"2024-04-10T03:02:38.142941+0000","last_became_peered":"2024-04-10T03:02:38.142941+0000","last_unstale":"2024-04-10T03:02:39.479682+0000","last_undegraded":"2024-04-10T03:02:39.479682+0000","last_fullsized":"2024-04-10T03:02:39.479682+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:25:57.656837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.145751+0000","last_change":"2024-04-10T03:02:38.321430+0000","last_active":"2024-04-10T03:02:39.145751+0000","last_peered":"2024-04-10T03:02:39.145751+0000","last_clean":"2024-04-10T03:02:39.145751+0000","last_became_active":"2024-04-10T03:02:38.321079+0000","last_became_peered":"2024-04-10T03:02:38.321079+0000","last_unstale":"2024-04-10T03:02:39.145751+0000","last_undegraded":"2024-04-10T03:02:39.145751+0000","last_fullsized":"2024-04-10T03:02:39.145751+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:39.342473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146604+0000","last_change":"2024-04-10T03:02:38.321148+0000","last_active":"2024-04-10T03:02:39.146604+0000","last_peered":"2024-04-10T03:02:39.146604+0000","last_clean":"2024-04-10T03:02:39.146604+0000","last_became_active":"2024-04-10T03:02:38.320773+0000","last_became_peered":"2024-04-10T03:02:38.320773+0000","last_unstale":"2024-04-10T03:02:39.146604+0000","last_undegraded":"2024-04-10T03:02:39.146604+0000","last_fullsized":"2024-04-10T03:02:39.146604+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:18:30.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146525+0000","last_change":"2024-04-10T03:02:38.141165+0000","last_active":"2024-04-10T03:02:39.146525+0000","last_peered":"2024-04-10T03:02:39.146525+0000","last_clean":"2024-04-10T03:02:39.146525+0000","last_became_active":"2024-04-10T03:02:38.140854+0000","last_became_peered":"2024-04-10T03:02:38.140854+0000","last_unstale":"2024-04-10T03:02:39.146525+0000","last_undegraded":"2024-04-10T03:02:39.146525+0000","last_fullsized":"2024-04-10T03:02:39.146525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:29:15.802525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146525+0000","last_change":"2024-04-10T03:02:08.306776+0000","last_active":"2024-04-10T03:02:39.146525+0000","last_peered":"2024-04-10T03:02:39.146525+0000","last_clean":"2024-04-10T03:02:39.146525+0000","last_became_active":"2024-04-10T03:02:08.306446+0000","last_became_peered":"2024-04-10T03:02:08.306446+0000","last_unstale":"2024-04-10T03:02:39.146525+0000","last_undegraded":"2024-04-10T03:02:39.146525+0000","last_fullsized":"2024-04-10T03:02:39.146525+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:40:01.838564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479634+0000","last_change":"2024-04-10T03:02:38.143477+0000","last_active":"2024-04-10T03:02:39.479634+0000","last_peered":"2024-04-10T03:02:39.479634+0000","last_clean":"2024-04-10T03:02:39.479634+0000","last_became_active":"2024-04-10T03:02:38.143077+0000","last_became_peered":"2024-04-10T03:02:38.143077+0000","last_unstale":"2024-04-10T03:02:39.479634+0000","last_undegraded":"2024-04-10T03:02:39.479634+0000","last_fullsized":"2024-04-10T03:02:39.479634+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:03.974803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479323+0000","last_change":"2024-04-10T03:02:38.552665+0000","last_active":"2024-04-10T03:02:39.479323+0000","last_peered":"2024-04-10T03:02:39.479323+0000","last_clean":"2024-04-10T03:02:39.479323+0000","last_became_active":"2024-04-10T03:02:38.552335+0000","last_became_peered":"2024-04-10T03:02:38.552335+0000","last_unstale":"2024-04-10T03:02:39.479323+0000","last_undegraded":"2024-04-10T03:02:39.479323+0000","last_fullsized":"2024-04-10T03:02:39.479323+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:54:39.380508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479768+0000","last_change":"2024-04-10T03:02:38.139183+0000","last_active":"2024-04-10T03:02:39.479768+0000","last_peered":"2024-04-10T03:02:39.479768+0000","last_clean":"2024-04-10T03:02:39.479768+0000","last_became_active":"2024-04-10T03:02:38.138885+0000","last_became_peered":"2024-04-10T03:02:38.138885+0000","last_unstale":"2024-04-10T03:02:39.479768+0000","last_undegraded":"2024-04-10T03:02:39.479768+0000","last_fullsized":"2024-04-10T03:02:39.479768+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:52:28.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.145767+0000","last_change":"2024-04-10T03:02:38.140035+0000","last_active":"2024-04-10T03:02:39.145767+0000","last_peered":"2024-04-10T03:02:39.145767+0000","last_clean":"2024-04-10T03:02:39.145767+0000","last_became_active":"2024-04-10T03:02:38.139760+0000","last_became_peered":"2024-04-10T03:02:38.139760+0000","last_unstale":"2024-04-10T03:02:39.145767+0000","last_undegraded":"2024-04-10T03:02:39.145767+0000","last_fullsized":"2024-04-10T03:02:39.145767+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:22.147412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.63100000000000001}]},{"osd":1,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57599999999999996}]},{"osd":2,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.622}]},{"osd":3,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.52900000000000003}]},{"osd":4,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.504},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.60899999999999999}]},{"osd":5,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.68700000000000006}]},{"osd":6,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.70999999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298544,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446608,"statfs":{"total":95995035648,"available":95689326592,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.503},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.59099999999999997}]},{"osd":1,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.73899999999999999}]},{"osd":2,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.71399999999999997}]},{"osd":3,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69899999999999995}]},{"osd":4,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.627}]},{"osd":5,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.75900000000000001}]},{"osd":7,"last update":"Wed Apr 10 03:03:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.79100000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.32900000000000001,"15min":0.30499999999999999},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.441},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.34599999999999997,"15min":0.318},"min":{"1min":0.252,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.47299999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.29099999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.52300000000000002}]},{"osd":3,"last update":"Wed Apr 10 03:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.45800000000000002,"15min":0.45700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.44600000000000001,"15min":0.435},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.504}]},{"osd":4,"last update":"Wed Apr 10 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56200000000000006,"15min":0.55900000000000005},"min":{"1min":0.41999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65000000000000002}]},{"osd":5,"last update":"Wed Apr 10 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81499999999999995,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.75800000000000001,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.63500000000000001}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.57999999999999996}]},{"osd":7,"last update":"Wed Apr 10 03:03:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55900000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Apr 10 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.33000000000000002,"15min":0.309},"min":{"1min":0.25700000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.33200000000000002},"min":{"1min":0.27200000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.32900000000000001}]},{"osd":2,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.40200000000000002},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.35299999999999998}]},{"osd":3,"last update":"Wed Apr 10 03:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.441,"15min":0.441},"min":{"1min":0.30099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.34300000000000003}]},{"osd":4,"last update":"Wed Apr 10 03:02:53 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.5}]},{"osd":5,"last update":"Wed Apr 10 03:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.44500000000000001}]},{"osd":6,"last update":"Wed Apr 10 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":3.359,"5min":3.359,"15min":3.359},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.48999999999999999}]},{"osd":7,"last update":"Wed Apr 10 03:03:06 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.50800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54500000000000004,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.54200000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.29999999999999999}]},{"osd":1,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.378,"15min":0.36099999999999999},"min":{"1min":0.245,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.56100000000000005,"5min":0.621,"15min":0.621},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41099999999999998,"15min":0.39700000000000002},"min":{"1min":0.30199999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57199999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.38300000000000001}]},{"osd":3,"last update":"Wed Apr 10 03:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.46100000000000002,"15min":0.46600000000000003},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.74199999999999999,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.436,"15min":0.442},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68300000000000005,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.35299999999999998}]},{"osd":4,"last update":"Wed Apr 10 03:02:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57499999999999996},"min":{"1min":0.44900000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66600000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57499999999999996,"15min":0.57999999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.44400000000000001}]},{"osd":5,"last update":"Wed Apr 10 03:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69199999999999995,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.52300000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:02:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.48899999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.38700000000000001,"15min":0.40200000000000002},"min":{"1min":0.25,"5min":0.221,"15min":0.221},"max":{"1min":0.47399999999999998,"5min":0.501,"15min":0.501},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.41299999999999998,"15min":0.42899999999999999},"min":{"1min":0.26200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57399999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.317}]},{"osd":1,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.41799999999999998,"15min":0.439},"min":{"1min":0.26300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.47799999999999998,"5min":0.752,"15min":0.752},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.45500000000000002,"15min":0.46899999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.58399999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.434,"15min":0.45500000000000002},"min":{"1min":0.317,"5min":0.25,"15min":0.25},"max":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48099999999999998,"15min":0.501},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.54000000000000004,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.41299999999999998}]},{"osd":4,"last update":"Wed Apr 10 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.39700000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.72599999999999998,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.69799999999999995}]},{"osd":5,"last update":"Wed Apr 10 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81999999999999995,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.71399999999999997}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":9.718,"5min":9.718,"15min":9.718},"last":0.67200000000000004}]},{"osd":7,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":9.9160000000000004,"5min":9.9160000000000004,"15min":9.9160000000000004},"last":0.60399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.504,"15min":0.498},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.67400000000000004,"5min":0.745,"15min":0.745},"last":0.52200000000000002}]},{"osd":1,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.51400000000000001,"15min":0.51000000000000001},"min":{"1min":0.39100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70499999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47599999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.45200000000000001}]},{"osd":3,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.40999999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.376}]},{"osd":5,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.312,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.41299999999999998,"15min":0.40300000000000002},"min":{"1min":0.33900000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.501}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.34200000000000003}]},{"osd":7,"last update":"Wed Apr 10 03:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.41699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70899999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.56699999999999995}]},{"osd":1,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.746,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.60099999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67900000000000005,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53800000000000003}]},{"osd":3,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.44800000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.64900000000000002}]},{"osd":4,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.42199999999999999,"15min":0.41999999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.45400000000000001,"15min":0.44600000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.58499999999999996}]},{"osd":6,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.503}]},{"osd":7,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:03:41.419 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:03:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:41 smithi102 ceph-mon[39651]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:41 smithi102 ceph-mon[39651]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:41.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:41 smithi102 ceph-mon[29301]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:41.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:41 smithi102 ceph-mon[29301]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:41.714 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:41.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:41 smithi152 ceph-mon[33523]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:41.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:41 smithi152 ceph-mon[33523]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:42.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:41 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-10T03:03:42.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:41 smithi102 systemd[1]: Stopped Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:42.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:41 smithi102 systemd[1]: Starting Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:42.483 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 podman[72425]: 2024-04-10 03:03:41.964138052 +0000 UTC m=+0.018523235 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-10T03:03:42.781 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-mon[29301]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:42.781 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-mon[39651]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:42.782 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 podman[72425]: 2024-04-10 03:03:42.486029579 +0000 UTC m=+0.540414747 volume create 7b9e94ffdc9fd282d5aeb5be05ba29cc2459fc2acee04ffb8bd397a0114d1811 2024-04-10T03:03:42.782 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 podman[72425]: 2024-04-10 03:03:42.561384493 +0000 UTC m=+0.615769661 container create 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:42 smithi152 ceph-mon[33523]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 podman[72425]: 2024-04-10 03:03:42.914233645 +0000 UTC m=+0.968618813 container init 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 podman[72425]: 2024-04-10 03:03:42.916874987 +0000 UTC m=+0.971260158 container start 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.929Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.929Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.930Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.102 port=9094 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.931Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.954Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:03:43.256 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.954Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:03:43.257 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.958Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-10T03:03:43.257 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:42.958Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-10T03:03:43.257 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:43 smithi102 bash[72425]: 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c 2024-04-10T03:03:43.257 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:43 smithi102 systemd[1]: Started Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[29301]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:03:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[29301]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:03:44.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[39651]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:03:44.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-mon[39651]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:03:44.351 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:03:44.353 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:03:44.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:44 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:44 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:44.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:44 smithi152 ceph-mon[33523]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:03:44.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:44 smithi152 ceph-mon[33523]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:03:45.056 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-04-10T03:03:44.088418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385500,"kb_used_data":6652,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575716,"statfs":{"total":767960285184,"available":765517533184,"internally_reserved":0,"allocated":6811648,"data_stored":3875712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002197"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479682+0000","last_change":"2024-04-10T03:02:38.143240+0000","last_active":"2024-04-10T03:02:39.479682+0000","last_peered":"2024-04-10T03:02:39.479682+0000","last_clean":"2024-04-10T03:02:39.479682+0000","last_became_active":"2024-04-10T03:02:38.142941+0000","last_became_peered":"2024-04-10T03:02:38.142941+0000","last_unstale":"2024-04-10T03:02:39.479682+0000","last_undegraded":"2024-04-10T03:02:39.479682+0000","last_fullsized":"2024-04-10T03:02:39.479682+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:25:57.656837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.145751+0000","last_change":"2024-04-10T03:02:38.321430+0000","last_active":"2024-04-10T03:02:39.145751+0000","last_peered":"2024-04-10T03:02:39.145751+0000","last_clean":"2024-04-10T03:02:39.145751+0000","last_became_active":"2024-04-10T03:02:38.321079+0000","last_became_peered":"2024-04-10T03:02:38.321079+0000","last_unstale":"2024-04-10T03:02:39.145751+0000","last_undegraded":"2024-04-10T03:02:39.145751+0000","last_fullsized":"2024-04-10T03:02:39.145751+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:39.342473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146604+0000","last_change":"2024-04-10T03:02:38.321148+0000","last_active":"2024-04-10T03:02:39.146604+0000","last_peered":"2024-04-10T03:02:39.146604+0000","last_clean":"2024-04-10T03:02:39.146604+0000","last_became_active":"2024-04-10T03:02:38.320773+0000","last_became_peered":"2024-04-10T03:02:38.320773+0000","last_unstale":"2024-04-10T03:02:39.146604+0000","last_undegraded":"2024-04-10T03:02:39.146604+0000","last_fullsized":"2024-04-10T03:02:39.146604+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:18:30.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146525+0000","last_change":"2024-04-10T03:02:38.141165+0000","last_active":"2024-04-10T03:02:39.146525+0000","last_peered":"2024-04-10T03:02:39.146525+0000","last_clean":"2024-04-10T03:02:39.146525+0000","last_became_active":"2024-04-10T03:02:38.140854+0000","last_became_peered":"2024-04-10T03:02:38.140854+0000","last_unstale":"2024-04-10T03:02:39.146525+0000","last_undegraded":"2024-04-10T03:02:39.146525+0000","last_fullsized":"2024-04-10T03:02:39.146525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:29:15.802525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.146525+0000","last_change":"2024-04-10T03:02:08.306776+0000","last_active":"2024-04-10T03:02:39.146525+0000","last_peered":"2024-04-10T03:02:39.146525+0000","last_clean":"2024-04-10T03:02:39.146525+0000","last_became_active":"2024-04-10T03:02:08.306446+0000","last_became_peered":"2024-04-10T03:02:08.306446+0000","last_unstale":"2024-04-10T03:02:39.146525+0000","last_undegraded":"2024-04-10T03:02:39.146525+0000","last_fullsized":"2024-04-10T03:02:39.146525+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:40:01.838564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479634+0000","last_change":"2024-04-10T03:02:38.143477+0000","last_active":"2024-04-10T03:02:39.479634+0000","last_peered":"2024-04-10T03:02:39.479634+0000","last_clean":"2024-04-10T03:02:39.479634+0000","last_became_active":"2024-04-10T03:02:38.143077+0000","last_became_peered":"2024-04-10T03:02:38.143077+0000","last_unstale":"2024-04-10T03:02:39.479634+0000","last_undegraded":"2024-04-10T03:02:39.479634+0000","last_fullsized":"2024-04-10T03:02:39.479634+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:03.974803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479323+0000","last_change":"2024-04-10T03:02:38.552665+0000","last_active":"2024-04-10T03:02:39.479323+0000","last_peered":"2024-04-10T03:02:39.479323+0000","last_clean":"2024-04-10T03:02:39.479323+0000","last_became_active":"2024-04-10T03:02:38.552335+0000","last_became_peered":"2024-04-10T03:02:38.552335+0000","last_unstale":"2024-04-10T03:02:39.479323+0000","last_undegraded":"2024-04-10T03:02:39.479323+0000","last_fullsized":"2024-04-10T03:02:39.479323+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:54:39.380508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.479768+0000","last_change":"2024-04-10T03:02:38.139183+0000","last_active":"2024-04-10T03:02:39.479768+0000","last_peered":"2024-04-10T03:02:39.479768+0000","last_clean":"2024-04-10T03:02:39.479768+0000","last_became_active":"2024-04-10T03:02:38.138885+0000","last_became_peered":"2024-04-10T03:02:38.138885+0000","last_unstale":"2024-04-10T03:02:39.479768+0000","last_undegraded":"2024-04-10T03:02:39.479768+0000","last_fullsized":"2024-04-10T03:02:39.479768+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:52:28.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-10T03:02:39.145767+0000","last_change":"2024-04-10T03:02:38.140035+0000","last_active":"2024-04-10T03:02:39.145767+0000","last_peered":"2024-04-10T03:02:39.145767+0000","last_clean":"2024-04-10T03:02:39.145767+0000","last_became_active":"2024-04-10T03:02:38.139760+0000","last_became_peered":"2024-04-10T03:02:38.139760+0000","last_unstale":"2024-04-10T03:02:39.145767+0000","last_undegraded":"2024-04-10T03:02:39.145767+0000","last_fullsized":"2024-04-10T03:02:39.145767+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:22.147412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.44600000000000001}]},{"osd":1,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.48199999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.435},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56200000000000006}]},{"osd":3,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.498},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51000000000000001}]},{"osd":4,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.55200000000000005}]},{"osd":5,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60199999999999998}]},{"osd":6,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57299999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495643,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298544,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446608,"statfs":{"total":95995035648,"available":95689326592,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.626},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.71399999999999997}]},{"osd":1,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.70399999999999996}]},{"osd":2,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.64900000000000002}]},{"osd":3,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.64000000000000001}]},{"osd":4,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.438},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.40100000000000002}]},{"osd":5,"last update":"Wed Apr 10 03:02:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.49099999999999999}]},{"osd":7,"last update":"Wed Apr 10 03:03:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56799999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.32900000000000001,"15min":0.30499999999999999},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.34599999999999997,"15min":0.318},"min":{"1min":0.252,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.253}]},{"osd":2,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.29099999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.29999999999999999}]},{"osd":3,"last update":"Wed Apr 10 03:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.45800000000000002,"15min":0.45700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":0.437},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.44600000000000001,"15min":0.435},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.41999999999999998}]},{"osd":4,"last update":"Wed Apr 10 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56200000000000006,"15min":0.55900000000000005},"min":{"1min":0.41999999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.54500000000000004,"15min":0.54000000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.56100000000000005}]},{"osd":5,"last update":"Wed Apr 10 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81499999999999995,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.75800000000000001,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.45400000000000001}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.53400000000000003}]},{"osd":7,"last update":"Wed Apr 10 03:03:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.52100000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Apr 10 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.33000000000000002,"15min":0.309},"min":{"1min":0.25700000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.33200000000000002},"min":{"1min":0.27200000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.32500000000000001}]},{"osd":2,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.63600000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.40200000000000002},"min":{"1min":0.26500000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.313}]},{"osd":3,"last update":"Wed Apr 10 03:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.441,"15min":0.441},"min":{"1min":0.30099999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.27900000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.34200000000000003}]},{"osd":4,"last update":"Wed Apr 10 03:02:53 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.74099999999999999}]},{"osd":5,"last update":"Wed Apr 10 03:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.77900000000000003}]},{"osd":6,"last update":"Wed Apr 10 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":3.359,"5min":3.359,"15min":3.359},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.76700000000000002}]},{"osd":7,"last update":"Wed Apr 10 03:03:06 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61899999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.33500000000000002},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54500000000000004,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.54200000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.35099999999999998}]},{"osd":1,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.378,"15min":0.36099999999999999},"min":{"1min":0.245,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.56100000000000005,"5min":0.621,"15min":0.621},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41099999999999998,"15min":0.39700000000000002},"min":{"1min":0.30199999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57199999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.92200000000000004}]},{"osd":3,"last update":"Wed Apr 10 03:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.46100000000000002,"15min":0.46600000000000003},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.74199999999999999,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.436,"15min":0.442},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68300000000000005,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.83699999999999997}]},{"osd":4,"last update":"Wed Apr 10 03:02:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57499999999999996},"min":{"1min":0.44900000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66600000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57499999999999996,"15min":0.57999999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.75600000000000001}]},{"osd":5,"last update":"Wed Apr 10 03:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69199999999999995,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.69199999999999995}]},{"osd":6,"last update":"Wed Apr 10 03:02:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.997}]},{"osd":7,"last update":"Wed Apr 10 03:03:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.874}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.38700000000000001,"15min":0.40200000000000002},"min":{"1min":0.25,"5min":0.221,"15min":0.221},"max":{"1min":0.47399999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.41299999999999998,"15min":0.42899999999999999},"min":{"1min":0.26200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57399999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.34899999999999998}]},{"osd":1,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.41799999999999998,"15min":0.439},"min":{"1min":0.26300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.47799999999999998,"5min":0.752,"15min":0.752},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.45500000000000002,"15min":0.46899999999999997},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.58399999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.47999999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:03:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.434,"15min":0.45500000000000002},"min":{"1min":0.317,"5min":0.25,"15min":0.25},"max":{"1min":0.53400000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.37},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.48099999999999998,"15min":0.501},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.54000000000000004,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.38700000000000001}]},{"osd":4,"last update":"Wed Apr 10 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.39700000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.72599999999999998,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.56100000000000005}]},{"osd":5,"last update":"Wed Apr 10 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81999999999999995,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.53500000000000003}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.623},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":9.718,"5min":9.718,"15min":9.718},"last":0.58499999999999996}]},{"osd":7,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.115,"5min":1.115,"15min":1.115},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":9.9160000000000004,"5min":9.9160000000000004,"15min":9.9160000000000004},"last":0.56999999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.504,"15min":0.498},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.67400000000000004,"5min":0.745,"15min":0.745},"last":0.52200000000000002}]},{"osd":1,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.51400000000000001,"15min":0.51000000000000001},"min":{"1min":0.39100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70499999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47599999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55100000000000005,"15min":0.55300000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.45200000000000001}]},{"osd":3,"last update":"Wed Apr 10 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.40999999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.72799999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.376}]},{"osd":5,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.312,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.41299999999999998,"15min":0.40300000000000002},"min":{"1min":0.33900000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.501}]},{"osd":6,"last update":"Wed Apr 10 03:02:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.34200000000000003}]},{"osd":7,"last update":"Wed Apr 10 03:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.41699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70899999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54600000000000004}]},{"osd":1,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.746,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.57599999999999996}]},{"osd":2,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67900000000000005,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.622}]},{"osd":3,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.44800000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59399999999999997}]},{"osd":4,"last update":"Wed Apr 10 03:03:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.42199999999999999,"15min":0.41999999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.45400000000000001,"15min":0.44600000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.51400000000000001}]},{"osd":6,"last update":"Wed Apr 10 03:03:40 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.33800000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.65900000000000003,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.47199999999999998,"15min":0.46300000000000002},"min":{"1min":0.36599999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.65400000000000003}]},{"osd":7,"last update":"Wed Apr 10 03:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:03:45.059 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-10T03:03:45.059 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-10T03:03:45.059 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-10T03:03:45.059 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:03:45.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:45 smithi152 ceph-mon[33523]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:45.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:45 smithi152 ceph-mon[33523]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:45.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:45 smithi102 ceph-mon[29301]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:45.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:45 smithi102 ceph-mon[29301]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:45.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:45 smithi102 ceph-mon[39651]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:45.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:45 smithi102 ceph-mon[39651]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:03:45.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:44.931Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000601547s 2024-04-10T03:03:45.313 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:46.262 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 systemd[1]: Stopping Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:46.577 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[60821]: ts=2024-04-10T03:03:46.304Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-10T03:03:46.578 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 podman[63947]: 2024-04-10 03:03:46.314496629 +0000 UTC m=+0.123611343 container died 8a1b92be2198d925bb6ca60e0ea402a98e429ce2d1dc7171e414c8b2f212bee0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 podman[63947]: 2024-04-10 03:03:46.77088272 +0000 UTC m=+0.579997419 container cleanup 8a1b92be2198d925bb6ca60e0ea402a98e429ce2d1dc7171e414c8b2f212bee0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:46 smithi152 bash[63947]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a 2024-04-10T03:03:47.175 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:03:47.175 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-10T03:03:47.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-mon[33523]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:47.404 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 podman[63961]: 2024-04-10 03:03:47.064190744 +0000 UTC m=+0.752404932 container remove 8a1b92be2198d925bb6ca60e0ea402a98e429ce2d1dc7171e414c8b2f212bee0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.404 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-10T03:03:47.404 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 systemd[1]: Stopped Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:47.405 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Consumed 1.018s CPU time. 2024-04-10T03:03:47.405 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 systemd[1]: Starting Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:03:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-mon[29301]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:47.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-mon[39651]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 podman[64062]: 2024-04-10 03:03:47.310742211 +0000 UTC m=+0.018832790 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 podman[64062]: 2024-04-10 03:03:47.406999579 +0000 UTC m=+0.115090149 container create a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 podman[64062]: 2024-04-10 03:03:47.617348301 +0000 UTC m=+0.325438871 container init a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 podman[64062]: 2024-04-10 03:03:47.619726733 +0000 UTC m=+0.327817314 container start a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.643Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.643Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.643Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi152 (none))" 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.643Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.643Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.644Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-10T03:03:47.662 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.645Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.646Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.646Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.578µs 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.648Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.011µs wal_replay_duration=437.816µs wbl_replay_duration=118ns total_replay_duration=475.554µs 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.649Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.649Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.649Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-10T03:03:47.663 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.663Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.418716ms db_storage=630ns remote_storage=1.061µs web_handler=247ns query_engine=541ns scrape=381.324µs scrape_sd=21.277µs notify=18.551µs notify_sd=5.378µs rules=13.740038ms tracing=5.277µs 2024-04-10T03:03:47.882 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-10T03:03:47.883 INFO:tasks.cephadm:Setup complete, yielding 2024-04-10T03:03:47.883 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-10T03:03:47.892 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T03:03:47.893 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- bash -c 'ceph fs volume create foo' 2024-04-10T03:03:47.907 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STOPPING 2024-04-10T03:03:47.907 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STOPPED 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STARTING 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Serving on http://:::9283 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STARTED 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STOPPING 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STOPPED 2024-04-10T03:03:47.908 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:47] ENGINE Bus STARTING 2024-04-10T03:03:48.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.664Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-10T03:03:48.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:03:47.664Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-10T03:03:48.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 bash[64062]: a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f 2024-04-10T03:03:48.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:03:47 smithi152 systemd[1]: Started Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:03:48.164 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/182168011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:48.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Serving on http://:::9283 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Bus STARTED 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Bus STOPPING 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Bus STOPPED 2024-04-10T03:03:48.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Bus STARTING 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Serving on http://:::9283 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:03:48] ENGINE Bus STARTED 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/182168011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:48 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:48.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/182168011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:03:48.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:48.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:48.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:48 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:03:49.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:49.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:49.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:49.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:49.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:49.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:49.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:49 smithi152 ceph-mon[33523]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:49.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:49.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[29301]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi102.front.sepia.ceph.com:9093"}]: dispatch 2024-04-10T03:03:49.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:03:49.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-10T03:03:49.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:03:49.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-10T03:03:49.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:49 smithi102 ceph-mon[39651]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:51.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:51 smithi152 ceph-mon[33523]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:51.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-10T03:03:51.230 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[29301]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[39651]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-10T03:03:51.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: osdmap e64: 8 total, 8 up, 8 in 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.498 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[29297]: 2024-04-10T03:03:52.163+0000 7f43f26bd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: osdmap e64: 8 total, 8 up, 8 in 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.499 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: osdmap e64: 8 total, 8 up, 8 in 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:52.885 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-10T03:03:53.168 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:03:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:03:52.933Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002423874s 2024-04-10T03:03:53.208 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: pgmap v46: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: osdmap e65: 8 total, 8 up, 8 in 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: osdmap e66: 8 total, 8 up, 8 in 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: fsmap foo:0 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: Saving service mds.foo spec with placement count:2 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.451 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: pgmap v46: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: osdmap e65: 8 total, 8 up, 8 in 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:53.452 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: osdmap e66: 8 total, 8 up, 8 in 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: fsmap foo:0 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: Saving service mds.foo spec with placement count:2 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:03:53.453 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: pgmap v46: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: osdmap e65: 8 total, 8 up, 8 in 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-10T03:03:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: osdmap e66: 8 total, 8 up, 8 in 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: fsmap foo:0 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: Saving service mds.foo spec with placement count:2 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:03:53.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:03:54.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:54 smithi152 ceph-mon[33523]: Deploying daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:03:54.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:54 smithi152 ceph-mon[33523]: osdmap e67: 8 total, 8 up, 8 in 2024-04-10T03:03:54.540 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:54 smithi102 ceph-mon[29301]: Deploying daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:03:54.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:54 smithi102 ceph-mon[29301]: osdmap e67: 8 total, 8 up, 8 in 2024-04-10T03:03:54.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:54 smithi102 ceph-mon[39651]: Deploying daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:03:54.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:54 smithi102 ceph-mon[39651]: osdmap e67: 8 total, 8 up, 8 in 2024-04-10T03:03:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:55 smithi152 ceph-mon[33523]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:55 smithi152 ceph-mon[33523]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:03:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:55 smithi152 ceph-mon[33523]: osdmap e68: 8 total, 8 up, 8 in 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[39651]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[39651]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[39651]: osdmap e68: 8 total, 8 up, 8 in 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[29301]: pgmap v50: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[29301]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:03:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:55 smithi102 ceph-mon[29301]: osdmap e68: 8 total, 8 up, 8 in 2024-04-10T03:03:55.684 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-10T03:03:56.774 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:03:57.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:57 smithi152 ceph-mon[33523]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:57.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:57 smithi102 ceph-mon[39651]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:57.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:57 smithi102 ceph-mon[29301]: pgmap v52: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:59.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[29301]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[39651]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:59.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:03:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:59 smithi152 ceph-mon[33523]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:03:59.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:03:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:03:59.466 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:03:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:03:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:03:59.592 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-10T03:03:59.628 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-10T03:03:59.628 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-10T03:03:59.629 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-10T03:03:59.882 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:boot 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: daemon mds.foo.smithi102.exkvif assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: Cluster is now healthy 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: fsmap foo:0 1 up:standby 2024-04-10T03:04:00.256 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:creating} 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: Deploying daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[29301]: daemon mds.foo.smithi102.exkvif is now active in filesystem foo as rank 0 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:boot 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: daemon mds.foo.smithi102.exkvif assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-10T03:04:00.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: fsmap foo:0 1 up:standby 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:creating} 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: Deploying daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:04:00.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:00 smithi102 ceph-mon[39651]: daemon mds.foo.smithi102.exkvif is now active in filesystem foo as rank 0 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:boot 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: daemon mds.foo.smithi102.exkvif assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: fsmap foo:0 1 up:standby 2024-04-10T03:04:00.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:04:00.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-10T03:04:00.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:creating} 2024-04-10T03:04:00.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:00.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: Deploying daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:04:00.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:00 smithi152 ceph-mon[33523]: daemon mds.foo.smithi102.exkvif is now active in filesystem foo as rank 0 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[29301]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[29301]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[39651]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[39651]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:01 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 2024-04-10T03:04:01.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:01 smithi152 ceph-mon[33523]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:01.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:01 smithi152 ceph-mon[33523]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:01.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:01 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 2024-04-10T03:04:01.783 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:02.486 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":21,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":3562317133},{"type":"v1","addr":"172.21.15.102:6801","nonce":3562317133}]},"active_addr":"172.21.15.102:6801/3562317133","active_change":"2024-04-10T03:02:36.066020+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.102:8443/","prometheus":"http://172.21.15.102:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1928572397}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":2965564602}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":2292901015}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1827401448}]}]} 2024-04-10T03:04:02.494 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-10T03:04:02.494 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-10T03:04:02.494 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:04:02.686 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:02] "GET /metrics HTTP/1.1" 200 203259 "" "Prometheus/2.43.0" 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3896796597' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:boot 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:04:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:02 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3896796597' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:boot 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3896796597' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:03.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-10T03:04:03.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:boot 2024-04-10T03:04:03.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:04:03.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:02 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:03.234 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:04.777 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[39651]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:04.778 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:04.778 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[39651]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-10T03:04:04.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[29301]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:04.778 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:04.779 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:04 smithi102 ceph-mon[29301]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-10T03:04:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:04 smithi152 ceph-mon[33523]: mds.? [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] up:active 2024-04-10T03:04:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:04 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:04 smithi152 ceph-mon[33523]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-10T03:04:05.213 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:05.213 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":68,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:03:54.183053+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-10T03:03:50.907839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-10T03:03:51.409886+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:04:05.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4189309921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:05.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:05.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:05.799 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4189309921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:05.799 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:05.799 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:05 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:05 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4189309921' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:05 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:05 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:06.647 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-10T03:04:06.647 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:04:06.955 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[39651]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[39651]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:standby 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[29301]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[29301]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:standby 2024-04-10T03:04:06.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:06 smithi102 ceph-mon[29301]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:06 smithi152 ceph-mon[33523]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:06 smithi152 ceph-mon[33523]: mds.? [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] up:standby 2024-04-10T03:04:07.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:06 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:04:07.343 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.515 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:08.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:09.245 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:09.246 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:09.246 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":68,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:03:54.183053+0000","last_up_change":"2024-04-10T03:02:04.553709+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-10T03:03:50.907839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-10T03:03:51.409886+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6803","nonce":3631766319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6805","nonce":3631766319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6809","nonce":3631766319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":3631766319},{"type":"v1","addr":"172.21.15.102:6807","nonce":3631766319}]},"public_addr":"172.21.15.102:6803/3631766319","cluster_addr":"172.21.15.102:6805/3631766319","heartbeat_back_addr":"172.21.15.102:6809/3631766319","heartbeat_front_addr":"172.21.15.102:6807/3631766319","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6811","nonce":2661864406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6813","nonce":2661864406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6817","nonce":2661864406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":2661864406},{"type":"v1","addr":"172.21.15.102:6815","nonce":2661864406}]},"public_addr":"172.21.15.102:6811/2661864406","cluster_addr":"172.21.15.102:6813/2661864406","heartbeat_back_addr":"172.21.15.102:6817/2661864406","heartbeat_front_addr":"172.21.15.102:6815/2661864406","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6819","nonce":2522010761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6821","nonce":2522010761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6825","nonce":2522010761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":2522010761},{"type":"v1","addr":"172.21.15.102:6823","nonce":2522010761}]},"public_addr":"172.21.15.102:6819/2522010761","cluster_addr":"172.21.15.102:6821/2522010761","heartbeat_back_addr":"172.21.15.102:6825/2522010761","heartbeat_front_addr":"172.21.15.102:6823/2522010761","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6827","nonce":248402932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6829","nonce":248402932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6833","nonce":248402932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":248402932},{"type":"v1","addr":"172.21.15.102:6831","nonce":248402932}]},"public_addr":"172.21.15.102:6827/248402932","cluster_addr":"172.21.15.102:6829/248402932","heartbeat_back_addr":"172.21.15.102:6833/248402932","heartbeat_front_addr":"172.21.15.102:6831/248402932","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:04:09.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:09 smithi152 ceph-mon[33523]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:09.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:09 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/971728905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:09.684 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:09 smithi102 ceph-mon[29301]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:09.684 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:09 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/971728905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:09.684 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:09 smithi102 ceph-mon[39651]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:09.684 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:09 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/971728905' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:04:10.571 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-10T03:04:10.572 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-10T03:04:10.572 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-10T03:04:10.573 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-10T03:04:10.573 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-10T03:04:10.575 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-10T03:04:10.575 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-10T03:04:10.576 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-10T03:04:11.084 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:11.157 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:11.159 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:11.161 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:11.498 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[29301]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:11.498 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.498 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.498 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.499 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[39651]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:11.499 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.499 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.499 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:11 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:11 smithi152 ceph-mon[33523]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:11.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:11 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:11 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:11.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:11 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:12.504 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:12] "GET /metrics HTTP/1.1" 200 239132 "" "Prometheus/2.43.0" 2024-04-10T03:04:12.528 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:12.959 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:12.961 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:13.391 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:13.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:13 smithi152 ceph-mon[33523]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:13.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:13 smithi102 ceph-mon[29301]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:13.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:13 smithi102 ceph-mon[39651]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-10T03:04:15.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:15 smithi152 ceph-mon[33523]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:15.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:15 smithi102 ceph-mon[29301]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:15.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:15 smithi102 ceph-mon[39651]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:17.683 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:17 smithi102 ceph-mon[29301]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:17.684 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:17 smithi102 ceph-mon[39651]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:17.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:17 smithi152 ceph-mon[33523]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:18.067 INFO:teuthology.orchestra.run.smithi102.stdout:231928234013 2024-04-10T03:04:18.068 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-10T03:04:19.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:19 smithi152 ceph-mon[33523]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:19.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:19.712 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:19 smithi102 ceph-mon[39651]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:19.712 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:19 smithi102 ceph-mon[29301]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-10T03:04:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:21 smithi102 ceph-mon[29301]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:21 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:21 smithi102 ceph-mon[39651]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:21 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:21.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:21 smithi152 ceph-mon[33523]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:21.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:21 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:22.529 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:22] "GET /metrics HTTP/1.1" 200 239094 "" "Prometheus/2.43.0" 2024-04-10T03:04:23.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:23 smithi102 ceph-mon[29301]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:23.593 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:23 smithi102 ceph-mon[39651]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:23.620 INFO:teuthology.orchestra.run.smithi102.stdout:107374182450 2024-04-10T03:04:23.620 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-10T03:04:23.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:23 smithi152 ceph-mon[33523]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:24.912 INFO:teuthology.orchestra.run.smithi102.stdout:77309411383 2024-04-10T03:04:24.913 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-10T03:04:25.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:25 smithi102 ceph-mon[29301]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:25 smithi102 ceph-mon[39651]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:25.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:25 smithi152 ceph-mon[33523]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:25.864 INFO:teuthology.orchestra.run.smithi102.stdout:34359738433 2024-04-10T03:04:25.865 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-10T03:04:26.545 INFO:teuthology.orchestra.run.smithi102.stdout:55834574908 2024-04-10T03:04:26.546 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.415 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:27.416 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.416 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.416 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:27.416 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:27.416 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:27 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:04:27.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:27 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:04:28.432 INFO:teuthology.orchestra.run.smithi102.stdout:128849018924 2024-04-10T03:04:28.433 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-10T03:04:28.838 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:28.842 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:28.843 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:28.843 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:28.878 INFO:teuthology.orchestra.run.smithi102.stdout:197568495653 2024-04-10T03:04:28.878 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-10T03:04:29.360 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:29 smithi102 ceph-mon[29301]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:29 smithi152 ceph-mon[33523]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:29.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:29.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:29 smithi102 ceph-mon[39651]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:31.472 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:31 smithi102 ceph-mon[29301]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:31.473 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:31 smithi102 ceph-mon[39651]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:31.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:31 smithi152 ceph-mon[33523]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:31.908 INFO:teuthology.orchestra.run.smithi102.stdout:163208757288 2024-04-10T03:04:31.909 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-10T03:04:32.257 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:32.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:32] "GET /metrics HTTP/1.1" 200 239094 "" "Prometheus/2.43.0" 2024-04-10T03:04:33.339 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:33.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:33 smithi102 ceph-mon[29301]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:33.545 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:33 smithi102 ceph-mon[39651]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:33 smithi152 ceph-mon[33523]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:34.720 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:34.722 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:34.908 INFO:teuthology.orchestra.run.smithi102.stdout:77309411385 2024-04-10T03:04:35.450 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:35 smithi102 ceph-mon[29301]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:35.450 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:35 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3136342770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:04:35.451 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:35 smithi102 ceph-mon[39651]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:35.451 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:35 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3136342770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:04:35.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:35 smithi152 ceph-mon[33523]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:35.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:35 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3136342770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-10T03:04:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-10T03:04:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-10T03:04:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:37.669 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[29301]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:37.669 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-10T03:04:37.669 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[29301]: osdmap e69: 8 total, 8 up, 8 in 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[39651]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-10T03:04:37.670 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:37 smithi102 ceph-mon[39651]: osdmap e69: 8 total, 8 up, 8 in 2024-04-10T03:04:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:37 smithi152 ceph-mon[33523]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-10T03:04:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-10T03:04:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:37 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-10T03:04:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:37 smithi152 ceph-mon[33523]: osdmap e69: 8 total, 8 up, 8 in 2024-04-10T03:04:38.296 INFO:teuthology.orchestra.run.smithi102.stdout:107374182452 2024-04-10T03:04:38.511 INFO:teuthology.orchestra.run.smithi102.stdout:231928234017 2024-04-10T03:04:38.669 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:38 smithi102 ceph-mon[39651]: osdmap e70: 8 total, 8 up, 8 in 2024-04-10T03:04:38.669 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:38 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4279552728' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:04:38.669 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:38 smithi102 ceph-mon[29301]: osdmap e70: 8 total, 8 up, 8 in 2024-04-10T03:04:38.669 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:38 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4279552728' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:04:38.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:38 smithi152 ceph-mon[33523]: osdmap e70: 8 total, 8 up, 8 in 2024-04-10T03:04:38.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:38 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4279552728' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:04:39.329 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234013 got 231928234017 for osd.7 2024-04-10T03:04:39.330 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:39 smithi152 ceph-mon[33523]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:39 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2616356106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:04:39.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:39.723 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:39 smithi102 ceph-mon[39651]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:39.723 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:39 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2616356106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:04:39.723 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:39 smithi102 ceph-mon[29301]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:39.723 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:39 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2616356106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:04:39.991 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182450 got 107374182452 for osd.3 2024-04-10T03:04:39.992 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:40.394 INFO:teuthology.orchestra.run.smithi102.stdout:55834574911 2024-04-10T03:04:41.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:41 smithi152 ceph-mon[33523]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:41.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:41 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2702354570' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:04:41.703 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:41 smithi102 ceph-mon[29301]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:41.703 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:41 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2702354570' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:04:41.703 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:41 smithi102 ceph-mon[39651]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:41.703 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:41 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2702354570' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:04:42.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:42] "GET /metrics HTTP/1.1" 200 239062 "" "Prometheus/2.43.0" 2024-04-10T03:04:42.917 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574908 got 55834574911 for osd.1 2024-04-10T03:04:42.918 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:43.675 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:43 smithi102 ceph-mon[39651]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:43.675 INFO:teuthology.orchestra.run.smithi102.stdout:34359738437 2024-04-10T03:04:43.675 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:43 smithi102 ceph-mon[29301]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:43 smithi152 ceph-mon[33523]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:43.912 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411385 for osd.2 2024-04-10T03:04:43.912 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:43.998 INFO:teuthology.orchestra.run.smithi102.stdout:197568495656 2024-04-10T03:04:44.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:44 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/570493811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:04:44.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:44 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/678925537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:04:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:44 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/570493811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:04:44.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:44 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/678925537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:04:44.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:44 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/570493811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:04:44.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:44 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/678925537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:04:45.755 INFO:teuthology.orchestra.run.smithi102.stdout:128849018929 2024-04-10T03:04:45.755 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:45 smithi102 ceph-mon[29301]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:45.755 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:45 smithi102 ceph-mon[39651]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:45 smithi152 ceph-mon[33523]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:45.912 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738437 for osd.0 2024-04-10T03:04:45.912 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:46.523 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018924 got 128849018929 for osd.4 2024-04-10T03:04:46.523 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:46.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:46 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4224580652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:04:46.718 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:46 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4224580652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:04:46.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:46 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4224580652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:04:46.928 INFO:teuthology.orchestra.run.smithi102.stdout:163208757291 2024-04-10T03:04:47.338 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495653 got 197568495656 for osd.6 2024-04-10T03:04:47.339 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:47 smithi152 ceph-mon[33523]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:47.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:47 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2815358217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:04:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:47 smithi102 ceph-mon[29301]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2815358217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:04:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:47 smithi102 ceph-mon[39651]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:47 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2815358217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:04:47.819 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757288 got 163208757291 for osd.5 2024-04-10T03:04:47.819 DEBUG:teuthology.parallel:result is None 2024-04-10T03:04:47.819 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-10T03:04:47.819 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:04:48.076 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:49.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:49 smithi152 ceph-mon[33523]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:49.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:49.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:49 smithi102 ceph-mon[29301]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:49 smithi102 ceph-mon[39651]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:49.849 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:49.851 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:04:50.616 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2024-04-10T03:04:48.101519+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386276,"kb_used_data":7420,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574940,"statfs":{"total":767960285184,"available":765516738560,"internally_reserved":0,"allocated":7598080,"data_stored":4286142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.724540"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378662+0000","last_change":"2024-04-10T03:03:53.199403+0000","last_active":"2024-04-10T03:04:36.378662+0000","last_peered":"2024-04-10T03:04:36.378662+0000","last_clean":"2024-04-10T03:04:36.378662+0000","last_became_active":"2024-04-10T03:03:53.199091+0000","last_became_peered":"2024-04-10T03:03:53.199091+0000","last_unstale":"2024-04-10T03:04:36.378662+0000","last_undegraded":"2024-04-10T03:04:36.378662+0000","last_fullsized":"2024-04-10T03:04:36.378662+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:07:04.347723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379215+0000","last_change":"2024-04-10T03:03:52.174045+0000","last_active":"2024-04-10T03:04:36.379215+0000","last_peered":"2024-04-10T03:04:36.379215+0000","last_clean":"2024-04-10T03:04:36.379215+0000","last_became_active":"2024-04-10T03:03:52.173281+0000","last_became_peered":"2024-04-10T03:03:52.173281+0000","last_unstale":"2024-04-10T03:04:36.379215+0000","last_undegraded":"2024-04-10T03:04:36.379215+0000","last_fullsized":"2024-04-10T03:04:36.379215+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:18:50.887661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381047+0000","last_change":"2024-04-10T03:03:53.196289+0000","last_active":"2024-04-10T03:04:36.381047+0000","last_peered":"2024-04-10T03:04:36.381047+0000","last_clean":"2024-04-10T03:04:36.381047+0000","last_became_active":"2024-04-10T03:03:53.196012+0000","last_became_peered":"2024-04-10T03:03:53.196012+0000","last_unstale":"2024-04-10T03:04:36.381047+0000","last_undegraded":"2024-04-10T03:04:36.381047+0000","last_fullsized":"2024-04-10T03:04:36.381047+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:46:28.369446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382107+0000","last_change":"2024-04-10T03:03:52.175141+0000","last_active":"2024-04-10T03:04:36.382107+0000","last_peered":"2024-04-10T03:04:36.382107+0000","last_clean":"2024-04-10T03:04:36.382107+0000","last_became_active":"2024-04-10T03:03:52.174710+0000","last_became_peered":"2024-04-10T03:03:52.174710+0000","last_unstale":"2024-04-10T03:04:36.382107+0000","last_undegraded":"2024-04-10T03:04:36.382107+0000","last_fullsized":"2024-04-10T03:04:36.382107+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:08:58.484193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378930+0000","last_change":"2024-04-10T03:03:53.197126+0000","last_active":"2024-04-10T03:04:36.378930+0000","last_peered":"2024-04-10T03:04:36.378930+0000","last_clean":"2024-04-10T03:04:36.378930+0000","last_became_active":"2024-04-10T03:03:53.196791+0000","last_became_peered":"2024-04-10T03:03:53.196791+0000","last_unstale":"2024-04-10T03:04:36.378930+0000","last_undegraded":"2024-04-10T03:04:36.378930+0000","last_fullsized":"2024-04-10T03:04:36.378930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:34:30.736007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377011+0000","last_change":"2024-04-10T03:03:52.169154+0000","last_active":"2024-04-10T03:04:36.377011+0000","last_peered":"2024-04-10T03:04:36.377011+0000","last_clean":"2024-04-10T03:04:36.377011+0000","last_became_active":"2024-04-10T03:03:52.168854+0000","last_became_peered":"2024-04-10T03:03:52.168854+0000","last_unstale":"2024-04-10T03:04:36.377011+0000","last_undegraded":"2024-04-10T03:04:36.377011+0000","last_fullsized":"2024-04-10T03:04:36.377011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:01:21.091245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379198+0000","last_change":"2024-04-10T03:03:53.200560+0000","last_active":"2024-04-10T03:04:36.379198+0000","last_peered":"2024-04-10T03:04:36.379198+0000","last_clean":"2024-04-10T03:04:36.379198+0000","last_became_active":"2024-04-10T03:03:53.200217+0000","last_became_peered":"2024-04-10T03:03:53.200217+0000","last_unstale":"2024-04-10T03:04:36.379198+0000","last_undegraded":"2024-04-10T03:04:36.379198+0000","last_fullsized":"2024-04-10T03:04:36.379198+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:19:22.060465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376896+0000","last_change":"2024-04-10T03:03:52.170933+0000","last_active":"2024-04-10T03:04:36.376896+0000","last_peered":"2024-04-10T03:04:36.376896+0000","last_clean":"2024-04-10T03:04:36.376896+0000","last_became_active":"2024-04-10T03:03:52.170616+0000","last_became_peered":"2024-04-10T03:03:52.170616+0000","last_unstale":"2024-04-10T03:04:36.376896+0000","last_undegraded":"2024-04-10T03:04:36.376896+0000","last_fullsized":"2024-04-10T03:04:36.376896+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:29.189824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376917+0000","last_change":"2024-04-10T03:03:53.193547+0000","last_active":"2024-04-10T03:04:36.376917+0000","last_peered":"2024-04-10T03:04:36.376917+0000","last_clean":"2024-04-10T03:04:36.376917+0000","last_became_active":"2024-04-10T03:03:53.193249+0000","last_became_peered":"2024-04-10T03:03:53.193249+0000","last_unstale":"2024-04-10T03:04:36.376917+0000","last_undegraded":"2024-04-10T03:04:36.376917+0000","last_fullsized":"2024-04-10T03:04:36.376917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:17:58.101131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379041+0000","last_change":"2024-04-10T03:03:52.175545+0000","last_active":"2024-04-10T03:04:36.379041+0000","last_peered":"2024-04-10T03:04:36.379041+0000","last_clean":"2024-04-10T03:04:36.379041+0000","last_became_active":"2024-04-10T03:03:52.175288+0000","last_became_peered":"2024-04-10T03:03:52.175288+0000","last_unstale":"2024-04-10T03:04:36.379041+0000","last_undegraded":"2024-04-10T03:04:36.379041+0000","last_fullsized":"2024-04-10T03:04:36.379041+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:45:19.900779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379043+0000","last_change":"2024-04-10T03:03:53.193384+0000","last_active":"2024-04-10T03:04:36.379043+0000","last_peered":"2024-04-10T03:04:36.379043+0000","last_clean":"2024-04-10T03:04:36.379043+0000","last_became_active":"2024-04-10T03:03:53.193048+0000","last_became_peered":"2024-04-10T03:03:53.193048+0000","last_unstale":"2024-04-10T03:04:36.379043+0000","last_undegraded":"2024-04-10T03:04:36.379043+0000","last_fullsized":"2024-04-10T03:04:36.379043+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:10:41.808720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379274+0000","last_change":"2024-04-10T03:03:52.171292+0000","last_active":"2024-04-10T03:04:36.379274+0000","last_peered":"2024-04-10T03:04:36.379274+0000","last_clean":"2024-04-10T03:04:36.379274+0000","last_became_active":"2024-04-10T03:03:52.171030+0000","last_became_peered":"2024-04-10T03:03:52.171030+0000","last_unstale":"2024-04-10T03:04:36.379274+0000","last_undegraded":"2024-04-10T03:04:36.379274+0000","last_fullsized":"2024-04-10T03:04:36.379274+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:10:28.134300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383763+0000","last_change":"2024-04-10T03:02:38.143240+0000","last_active":"2024-04-10T03:04:36.383763+0000","last_peered":"2024-04-10T03:04:36.383763+0000","last_clean":"2024-04-10T03:04:36.383763+0000","last_became_active":"2024-04-10T03:02:38.142941+0000","last_became_peered":"2024-04-10T03:02:38.142941+0000","last_unstale":"2024-04-10T03:04:36.383763+0000","last_undegraded":"2024-04-10T03:04:36.383763+0000","last_fullsized":"2024-04-10T03:04:36.383763+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:25:57.656837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382234+0000","last_change":"2024-04-10T03:03:53.201926+0000","last_active":"2024-04-10T03:04:36.382234+0000","last_peered":"2024-04-10T03:04:36.382234+0000","last_clean":"2024-04-10T03:04:36.382234+0000","last_became_active":"2024-04-10T03:03:53.201614+0000","last_became_peered":"2024-04-10T03:03:53.201614+0000","last_unstale":"2024-04-10T03:04:36.382234+0000","last_undegraded":"2024-04-10T03:04:36.382234+0000","last_fullsized":"2024-04-10T03:04:36.382234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:06:56.966117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379568+0000","last_change":"2024-04-10T03:02:38.321430+0000","last_active":"2024-04-10T03:04:36.379568+0000","last_peered":"2024-04-10T03:04:36.379568+0000","last_clean":"2024-04-10T03:04:36.379568+0000","last_became_active":"2024-04-10T03:02:38.321079+0000","last_became_peered":"2024-04-10T03:02:38.321079+0000","last_unstale":"2024-04-10T03:04:36.379568+0000","last_undegraded":"2024-04-10T03:04:36.379568+0000","last_fullsized":"2024-04-10T03:04:36.379568+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:39.342473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382310+0000","last_change":"2024-04-10T03:03:52.182258+0000","last_active":"2024-04-10T03:04:36.382310+0000","last_peered":"2024-04-10T03:04:36.382310+0000","last_clean":"2024-04-10T03:04:36.382310+0000","last_became_active":"2024-04-10T03:03:52.181986+0000","last_became_peered":"2024-04-10T03:03:52.181986+0000","last_unstale":"2024-04-10T03:04:36.382310+0000","last_undegraded":"2024-04-10T03:04:36.382310+0000","last_fullsized":"2024-04-10T03:04:36.382310+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:32:34.330120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378909+0000","last_change":"2024-04-10T03:03:53.205741+0000","last_active":"2024-04-10T03:04:36.378909+0000","last_peered":"2024-04-10T03:04:36.378909+0000","last_clean":"2024-04-10T03:04:36.378909+0000","last_became_active":"2024-04-10T03:03:53.205418+0000","last_became_peered":"2024-04-10T03:03:53.205418+0000","last_unstale":"2024-04-10T03:04:36.378909+0000","last_undegraded":"2024-04-10T03:04:36.378909+0000","last_fullsized":"2024-04-10T03:04:36.378909+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:37:35.498166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378646+0000","last_change":"2024-04-10T03:03:52.176007+0000","last_active":"2024-04-10T03:04:36.378646+0000","last_peered":"2024-04-10T03:04:36.378646+0000","last_clean":"2024-04-10T03:04:36.378646+0000","last_became_active":"2024-04-10T03:03:52.175722+0000","last_became_peered":"2024-04-10T03:03:52.175722+0000","last_unstale":"2024-04-10T03:04:36.378646+0000","last_undegraded":"2024-04-10T03:04:36.378646+0000","last_fullsized":"2024-04-10T03:04:36.378646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:48.316168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527086+0000","last_change":"2024-04-10T03:02:38.321148+0000","last_active":"2024-04-10T03:04:36.527086+0000","last_peered":"2024-04-10T03:04:36.527086+0000","last_clean":"2024-04-10T03:04:36.527086+0000","last_became_active":"2024-04-10T03:02:38.320773+0000","last_became_peered":"2024-04-10T03:02:38.320773+0000","last_unstale":"2024-04-10T03:04:36.527086+0000","last_undegraded":"2024-04-10T03:04:36.527086+0000","last_fullsized":"2024-04-10T03:04:36.527086+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:18:30.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067157+0000","last_change":"2024-04-10T03:04:38.067157+0000","last_active":"2024-04-10T03:04:38.067157+0000","last_peered":"2024-04-10T03:04:38.067157+0000","last_clean":"2024-04-10T03:04:38.067157+0000","last_became_active":"2024-04-10T03:04:38.066711+0000","last_became_peered":"2024-04-10T03:04:38.066711+0000","last_unstale":"2024-04-10T03:04:38.067157+0000","last_undegraded":"2024-04-10T03:04:38.067157+0000","last_fullsized":"2024-04-10T03:04:38.067157+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:14:54.573143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378706+0000","last_change":"2024-04-10T03:03:52.177504+0000","last_active":"2024-04-10T03:04:36.378706+0000","last_peered":"2024-04-10T03:04:36.378706+0000","last_clean":"2024-04-10T03:04:36.378706+0000","last_became_active":"2024-04-10T03:03:52.177241+0000","last_became_peered":"2024-04-10T03:03:52.177241+0000","last_unstale":"2024-04-10T03:04:36.378706+0000","last_undegraded":"2024-04-10T03:04:36.378706+0000","last_fullsized":"2024-04-10T03:04:36.378706+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:21:52.152273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526782+0000","last_change":"2024-04-10T03:02:38.141165+0000","last_active":"2024-04-10T03:04:36.526782+0000","last_peered":"2024-04-10T03:04:36.526782+0000","last_clean":"2024-04-10T03:04:36.526782+0000","last_became_active":"2024-04-10T03:02:38.140854+0000","last_became_peered":"2024-04-10T03:02:38.140854+0000","last_unstale":"2024-04-10T03:04:36.526782+0000","last_undegraded":"2024-04-10T03:04:36.526782+0000","last_fullsized":"2024-04-10T03:04:36.526782+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:29:15.802525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383683+0000","last_change":"2024-04-10T03:03:53.201722+0000","last_active":"2024-04-10T03:04:36.383683+0000","last_peered":"2024-04-10T03:04:36.383683+0000","last_clean":"2024-04-10T03:04:36.383683+0000","last_became_active":"2024-04-10T03:03:53.201394+0000","last_became_peered":"2024-04-10T03:03:53.201394+0000","last_unstale":"2024-04-10T03:04:36.383683+0000","last_undegraded":"2024-04-10T03:04:36.383683+0000","last_fullsized":"2024-04-10T03:04:36.383683+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:16.107170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381359+0000","last_change":"2024-04-10T03:03:52.177805+0000","last_active":"2024-04-10T03:04:36.381359+0000","last_peered":"2024-04-10T03:04:36.381359+0000","last_clean":"2024-04-10T03:04:36.381359+0000","last_became_active":"2024-04-10T03:03:52.174720+0000","last_became_peered":"2024-04-10T03:03:52.174720+0000","last_unstale":"2024-04-10T03:04:36.381359+0000","last_undegraded":"2024-04-10T03:04:36.381359+0000","last_fullsized":"2024-04-10T03:04:36.381359+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:09:38.594358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526786+0000","last_change":"2024-04-10T03:02:08.306776+0000","last_active":"2024-04-10T03:04:36.526786+0000","last_peered":"2024-04-10T03:04:36.526786+0000","last_clean":"2024-04-10T03:04:36.526786+0000","last_became_active":"2024-04-10T03:02:08.306446+0000","last_became_peered":"2024-04-10T03:02:08.306446+0000","last_unstale":"2024-04-10T03:04:36.526786+0000","last_undegraded":"2024-04-10T03:04:36.526786+0000","last_fullsized":"2024-04-10T03:04:36.526786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:40:01.838564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376787+0000","last_change":"2024-04-10T03:02:38.143477+0000","last_active":"2024-04-10T03:04:36.376787+0000","last_peered":"2024-04-10T03:04:36.376787+0000","last_clean":"2024-04-10T03:04:36.376787+0000","last_became_active":"2024-04-10T03:02:38.143077+0000","last_became_peered":"2024-04-10T03:02:38.143077+0000","last_unstale":"2024-04-10T03:04:36.376787+0000","last_undegraded":"2024-04-10T03:04:36.376787+0000","last_fullsized":"2024-04-10T03:04:36.376787+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:03.974803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379545+0000","last_change":"2024-04-10T03:03:53.196065+0000","last_active":"2024-04-10T03:04:36.379545+0000","last_peered":"2024-04-10T03:04:36.379545+0000","last_clean":"2024-04-10T03:04:36.379545+0000","last_became_active":"2024-04-10T03:03:53.195767+0000","last_became_peered":"2024-04-10T03:03:53.195767+0000","last_unstale":"2024-04-10T03:04:36.379545+0000","last_undegraded":"2024-04-10T03:04:36.379545+0000","last_fullsized":"2024-04-10T03:04:36.379545+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:03:51.593233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377578+0000","last_change":"2024-04-10T03:03:52.171011+0000","last_active":"2024-04-10T03:04:36.377578+0000","last_peered":"2024-04-10T03:04:36.377578+0000","last_clean":"2024-04-10T03:04:36.377578+0000","last_became_active":"2024-04-10T03:03:52.170746+0000","last_became_peered":"2024-04-10T03:03:52.170746+0000","last_unstale":"2024-04-10T03:04:36.377578+0000","last_undegraded":"2024-04-10T03:04:36.377578+0000","last_fullsized":"2024-04-10T03:04:36.377578+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:39:06.035417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376736+0000","last_change":"2024-04-10T03:02:38.552665+0000","last_active":"2024-04-10T03:04:36.376736+0000","last_peered":"2024-04-10T03:04:36.376736+0000","last_clean":"2024-04-10T03:04:36.376736+0000","last_became_active":"2024-04-10T03:02:38.552335+0000","last_became_peered":"2024-04-10T03:02:38.552335+0000","last_unstale":"2024-04-10T03:04:36.376736+0000","last_undegraded":"2024-04-10T03:04:36.376736+0000","last_fullsized":"2024-04-10T03:04:36.376736+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:54:39.380508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378571+0000","last_change":"2024-04-10T03:03:53.200043+0000","last_active":"2024-04-10T03:04:36.378571+0000","last_peered":"2024-04-10T03:04:36.378571+0000","last_clean":"2024-04-10T03:04:36.378571+0000","last_became_active":"2024-04-10T03:03:53.199780+0000","last_became_peered":"2024-04-10T03:03:53.199780+0000","last_unstale":"2024-04-10T03:04:36.378571+0000","last_undegraded":"2024-04-10T03:04:36.378571+0000","last_fullsized":"2024-04-10T03:04:36.378571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:01:12.981096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379127+0000","last_change":"2024-04-10T03:03:52.176230+0000","last_active":"2024-04-10T03:04:36.379127+0000","last_peered":"2024-04-10T03:04:36.379127+0000","last_clean":"2024-04-10T03:04:36.379127+0000","last_became_active":"2024-04-10T03:03:52.175660+0000","last_became_peered":"2024-04-10T03:03:52.175660+0000","last_unstale":"2024-04-10T03:04:36.379127+0000","last_undegraded":"2024-04-10T03:04:36.379127+0000","last_fullsized":"2024-04-10T03:04:36.379127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:41:32.982610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376661+0000","last_change":"2024-04-10T03:02:38.139183+0000","last_active":"2024-04-10T03:04:36.376661+0000","last_peered":"2024-04-10T03:04:36.376661+0000","last_clean":"2024-04-10T03:04:36.376661+0000","last_became_active":"2024-04-10T03:02:38.138885+0000","last_became_peered":"2024-04-10T03:02:38.138885+0000","last_unstale":"2024-04-10T03:04:36.376661+0000","last_undegraded":"2024-04-10T03:04:36.376661+0000","last_fullsized":"2024-04-10T03:04:36.376661+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:52:28.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377886+0000","last_change":"2024-04-10T03:03:53.198290+0000","last_active":"2024-04-10T03:04:36.377886+0000","last_peered":"2024-04-10T03:04:36.377886+0000","last_clean":"2024-04-10T03:04:36.377886+0000","last_became_active":"2024-04-10T03:03:53.198009+0000","last_became_peered":"2024-04-10T03:03:53.198009+0000","last_unstale":"2024-04-10T03:04:36.377886+0000","last_undegraded":"2024-04-10T03:04:36.377886+0000","last_fullsized":"2024-04-10T03:04:36.377886+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:50:00.001489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376897+0000","last_change":"2024-04-10T03:03:52.173812+0000","last_active":"2024-04-10T03:04:36.376897+0000","last_peered":"2024-04-10T03:04:36.376897+0000","last_clean":"2024-04-10T03:04:36.376897+0000","last_became_active":"2024-04-10T03:03:52.173468+0000","last_became_peered":"2024-04-10T03:03:52.173468+0000","last_unstale":"2024-04-10T03:04:36.376897+0000","last_undegraded":"2024-04-10T03:04:36.376897+0000","last_fullsized":"2024-04-10T03:04:36.376897+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:08:33.209359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378503+0000","last_change":"2024-04-10T03:02:38.140035+0000","last_active":"2024-04-10T03:04:36.378503+0000","last_peered":"2024-04-10T03:04:36.378503+0000","last_clean":"2024-04-10T03:04:36.378503+0000","last_became_active":"2024-04-10T03:02:38.139760+0000","last_became_peered":"2024-04-10T03:02:38.139760+0000","last_unstale":"2024-04-10T03:04:36.378503+0000","last_undegraded":"2024-04-10T03:04:36.378503+0000","last_fullsized":"2024-04-10T03:04:36.378503+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:22.147412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379469+0000","last_change":"2024-04-10T03:03:53.200087+0000","last_active":"2024-04-10T03:04:36.379469+0000","last_peered":"2024-04-10T03:04:36.379469+0000","last_clean":"2024-04-10T03:04:36.379469+0000","last_became_active":"2024-04-10T03:03:53.199771+0000","last_became_peered":"2024-04-10T03:03:53.199771+0000","last_unstale":"2024-04-10T03:04:36.379469+0000","last_undegraded":"2024-04-10T03:04:36.379469+0000","last_fullsized":"2024-04-10T03:04:36.379469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:10:48.839174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382316+0000","last_change":"2024-04-10T03:03:52.178981+0000","last_active":"2024-04-10T03:04:36.382316+0000","last_peered":"2024-04-10T03:04:36.382316+0000","last_clean":"2024-04-10T03:04:36.382316+0000","last_became_active":"2024-04-10T03:03:52.178210+0000","last_became_peered":"2024-04-10T03:03:52.178210+0000","last_unstale":"2024-04-10T03:04:36.382316+0000","last_undegraded":"2024-04-10T03:04:36.382316+0000","last_fullsized":"2024-04-10T03:04:36.382316+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:07:53.916420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379055+0000","last_change":"2024-04-10T03:03:53.200095+0000","last_active":"2024-04-10T03:04:36.379055+0000","last_peered":"2024-04-10T03:04:36.379055+0000","last_clean":"2024-04-10T03:04:36.379055+0000","last_became_active":"2024-04-10T03:03:53.199779+0000","last_became_peered":"2024-04-10T03:03:53.199779+0000","last_unstale":"2024-04-10T03:04:36.379055+0000","last_undegraded":"2024-04-10T03:04:36.379055+0000","last_fullsized":"2024-04-10T03:04:36.379055+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:16:44.877622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378771+0000","last_change":"2024-04-10T03:03:52.172894+0000","last_active":"2024-04-10T03:04:36.378771+0000","last_peered":"2024-04-10T03:04:36.378771+0000","last_clean":"2024-04-10T03:04:36.378771+0000","last_became_active":"2024-04-10T03:03:52.172669+0000","last_became_peered":"2024-04-10T03:03:52.172669+0000","last_unstale":"2024-04-10T03:04:36.378771+0000","last_undegraded":"2024-04-10T03:04:36.378771+0000","last_fullsized":"2024-04-10T03:04:36.378771+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:06:44.554916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067113+0000","last_change":"2024-04-10T03:04:38.067113+0000","last_active":"2024-04-10T03:04:38.067113+0000","last_peered":"2024-04-10T03:04:38.067113+0000","last_clean":"2024-04-10T03:04:38.067113+0000","last_became_active":"2024-04-10T03:04:38.066631+0000","last_became_peered":"2024-04-10T03:04:38.066631+0000","last_unstale":"2024-04-10T03:04:38.067113+0000","last_undegraded":"2024-04-10T03:04:38.067113+0000","last_fullsized":"2024-04-10T03:04:38.067113+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:37:07.088775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383497+0000","last_change":"2024-04-10T03:03:52.176489+0000","last_active":"2024-04-10T03:04:36.383497+0000","last_peered":"2024-04-10T03:04:36.383497+0000","last_clean":"2024-04-10T03:04:36.383497+0000","last_became_active":"2024-04-10T03:03:52.176198+0000","last_became_peered":"2024-04-10T03:03:52.176198+0000","last_unstale":"2024-04-10T03:04:36.383497+0000","last_undegraded":"2024-04-10T03:04:36.383497+0000","last_fullsized":"2024-04-10T03:04:36.383497+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:35:37.911620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378905+0000","last_change":"2024-04-10T03:03:53.197184+0000","last_active":"2024-04-10T03:04:36.378905+0000","last_peered":"2024-04-10T03:04:36.378905+0000","last_clean":"2024-04-10T03:04:36.378905+0000","last_became_active":"2024-04-10T03:03:53.196842+0000","last_became_peered":"2024-04-10T03:03:53.196842+0000","last_unstale":"2024-04-10T03:04:36.378905+0000","last_undegraded":"2024-04-10T03:04:36.378905+0000","last_fullsized":"2024-04-10T03:04:36.378905+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:42:29.861897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381519+0000","last_change":"2024-04-10T03:03:52.181581+0000","last_active":"2024-04-10T03:04:36.381519+0000","last_peered":"2024-04-10T03:04:36.381519+0000","last_clean":"2024-04-10T03:04:36.381519+0000","last_became_active":"2024-04-10T03:03:52.181307+0000","last_became_peered":"2024-04-10T03:03:52.181307+0000","last_unstale":"2024-04-10T03:04:36.381519+0000","last_undegraded":"2024-04-10T03:04:36.381519+0000","last_fullsized":"2024-04-10T03:04:36.381519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:39:43.513327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379350+0000","last_change":"2024-04-10T03:03:53.196706+0000","last_active":"2024-04-10T03:04:36.379350+0000","last_peered":"2024-04-10T03:04:36.379350+0000","last_clean":"2024-04-10T03:04:36.379350+0000","last_became_active":"2024-04-10T03:03:53.196450+0000","last_became_peered":"2024-04-10T03:03:53.196450+0000","last_unstale":"2024-04-10T03:04:36.379350+0000","last_undegraded":"2024-04-10T03:04:36.379350+0000","last_fullsized":"2024-04-10T03:04:36.379350+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:38:56.781064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376821+0000","last_change":"2024-04-10T03:03:52.175356+0000","last_active":"2024-04-10T03:04:36.376821+0000","last_peered":"2024-04-10T03:04:36.376821+0000","last_clean":"2024-04-10T03:04:36.376821+0000","last_became_active":"2024-04-10T03:03:52.175063+0000","last_became_peered":"2024-04-10T03:03:52.175063+0000","last_unstale":"2024-04-10T03:04:36.376821+0000","last_undegraded":"2024-04-10T03:04:36.376821+0000","last_fullsized":"2024-04-10T03:04:36.376821+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:13:07.527860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383436+0000","last_change":"2024-04-10T03:03:53.200545+0000","last_active":"2024-04-10T03:04:36.383436+0000","last_peered":"2024-04-10T03:04:36.383436+0000","last_clean":"2024-04-10T03:04:36.383436+0000","last_became_active":"2024-04-10T03:03:53.200173+0000","last_became_peered":"2024-04-10T03:03:53.200173+0000","last_unstale":"2024-04-10T03:04:36.383436+0000","last_undegraded":"2024-04-10T03:04:36.383436+0000","last_fullsized":"2024-04-10T03:04:36.383436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:53:35.906724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527172+0000","last_change":"2024-04-10T03:03:52.170665+0000","last_active":"2024-04-10T03:04:36.527172+0000","last_peered":"2024-04-10T03:04:36.527172+0000","last_clean":"2024-04-10T03:04:36.527172+0000","last_became_active":"2024-04-10T03:03:52.170385+0000","last_became_peered":"2024-04-10T03:03:52.170385+0000","last_unstale":"2024-04-10T03:04:36.527172+0000","last_undegraded":"2024-04-10T03:04:36.527172+0000","last_fullsized":"2024-04-10T03:04:36.527172+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:36:28.794484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378728+0000","last_change":"2024-04-10T03:03:53.199941+0000","last_active":"2024-04-10T03:04:36.378728+0000","last_peered":"2024-04-10T03:04:36.378728+0000","last_clean":"2024-04-10T03:04:36.378728+0000","last_became_active":"2024-04-10T03:03:53.199621+0000","last_became_peered":"2024-04-10T03:03:53.199621+0000","last_unstale":"2024-04-10T03:04:36.378728+0000","last_undegraded":"2024-04-10T03:04:36.378728+0000","last_fullsized":"2024-04-10T03:04:36.378728+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:24:25.332895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527115+0000","last_change":"2024-04-10T03:03:52.176385+0000","last_active":"2024-04-10T03:04:36.527115+0000","last_peered":"2024-04-10T03:04:36.527115+0000","last_clean":"2024-04-10T03:04:36.527115+0000","last_became_active":"2024-04-10T03:03:52.176122+0000","last_became_peered":"2024-04-10T03:03:52.176122+0000","last_unstale":"2024-04-10T03:04:36.527115+0000","last_undegraded":"2024-04-10T03:04:36.527115+0000","last_fullsized":"2024-04-10T03:04:36.527115+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:15:20.134402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376652+0000","last_change":"2024-04-10T03:03:53.200529+0000","last_active":"2024-04-10T03:04:36.376652+0000","last_peered":"2024-04-10T03:04:36.376652+0000","last_clean":"2024-04-10T03:04:36.376652+0000","last_became_active":"2024-04-10T03:03:53.199080+0000","last_became_peered":"2024-04-10T03:03:53.199080+0000","last_unstale":"2024-04-10T03:04:36.376652+0000","last_undegraded":"2024-04-10T03:04:36.376652+0000","last_fullsized":"2024-04-10T03:04:36.376652+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T15:00:49.737785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527202+0000","last_change":"2024-04-10T03:03:52.178341+0000","last_active":"2024-04-10T03:04:36.527202+0000","last_peered":"2024-04-10T03:04:36.527202+0000","last_clean":"2024-04-10T03:04:36.527202+0000","last_became_active":"2024-04-10T03:03:52.178082+0000","last_became_peered":"2024-04-10T03:03:52.178082+0000","last_unstale":"2024-04-10T03:04:36.527202+0000","last_undegraded":"2024-04-10T03:04:36.527202+0000","last_fullsized":"2024-04-10T03:04:36.527202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:18:59.101964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382386+0000","last_change":"2024-04-10T03:03:53.205118+0000","last_active":"2024-04-10T03:04:36.382386+0000","last_peered":"2024-04-10T03:04:36.382386+0000","last_clean":"2024-04-10T03:04:36.382386+0000","last_became_active":"2024-04-10T03:03:53.204845+0000","last_became_peered":"2024-04-10T03:03:53.204845+0000","last_unstale":"2024-04-10T03:04:36.382386+0000","last_undegraded":"2024-04-10T03:04:36.382386+0000","last_fullsized":"2024-04-10T03:04:36.382386+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:23:30.856777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383866+0000","last_change":"2024-04-10T03:03:52.174189+0000","last_active":"2024-04-10T03:04:36.383866+0000","last_peered":"2024-04-10T03:04:36.383866+0000","last_clean":"2024-04-10T03:04:36.383866+0000","last_became_active":"2024-04-10T03:03:52.173057+0000","last_became_peered":"2024-04-10T03:03:52.173057+0000","last_unstale":"2024-04-10T03:04:36.383866+0000","last_undegraded":"2024-04-10T03:04:36.383866+0000","last_fullsized":"2024-04-10T03:04:36.383866+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:30:45.451822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379127+0000","last_change":"2024-04-10T03:03:53.200354+0000","last_active":"2024-04-10T03:04:36.379127+0000","last_peered":"2024-04-10T03:04:36.379127+0000","last_clean":"2024-04-10T03:04:36.379127+0000","last_became_active":"2024-04-10T03:03:53.200095+0000","last_became_peered":"2024-04-10T03:03:53.200095+0000","last_unstale":"2024-04-10T03:04:36.379127+0000","last_undegraded":"2024-04-10T03:04:36.379127+0000","last_fullsized":"2024-04-10T03:04:36.379127+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:33:26.829187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526814+0000","last_change":"2024-04-10T03:03:52.171594+0000","last_active":"2024-04-10T03:04:36.526814+0000","last_peered":"2024-04-10T03:04:36.526814+0000","last_clean":"2024-04-10T03:04:36.526814+0000","last_became_active":"2024-04-10T03:03:52.171314+0000","last_became_peered":"2024-04-10T03:03:52.171314+0000","last_unstale":"2024-04-10T03:04:36.526814+0000","last_undegraded":"2024-04-10T03:04:36.526814+0000","last_fullsized":"2024-04-10T03:04:36.526814+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:37:55.120481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376892+0000","last_change":"2024-04-10T03:03:53.201997+0000","last_active":"2024-04-10T03:04:36.376892+0000","last_peered":"2024-04-10T03:04:36.376892+0000","last_clean":"2024-04-10T03:04:36.376892+0000","last_became_active":"2024-04-10T03:03:53.201679+0000","last_became_peered":"2024-04-10T03:03:53.201679+0000","last_unstale":"2024-04-10T03:04:36.376892+0000","last_undegraded":"2024-04-10T03:04:36.376892+0000","last_fullsized":"2024-04-10T03:04:36.376892+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:26:50.320247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379350+0000","last_change":"2024-04-10T03:03:52.179168+0000","last_active":"2024-04-10T03:04:36.379350+0000","last_peered":"2024-04-10T03:04:36.379350+0000","last_clean":"2024-04-10T03:04:36.379350+0000","last_became_active":"2024-04-10T03:03:52.178906+0000","last_became_peered":"2024-04-10T03:03:52.178906+0000","last_unstale":"2024-04-10T03:04:36.379350+0000","last_undegraded":"2024-04-10T03:04:36.379350+0000","last_fullsized":"2024-04-10T03:04:36.379350+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:33:10.623935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382482+0000","last_change":"2024-04-10T03:03:53.204493+0000","last_active":"2024-04-10T03:04:36.382482+0000","last_peered":"2024-04-10T03:04:36.382482+0000","last_clean":"2024-04-10T03:04:36.382482+0000","last_became_active":"2024-04-10T03:03:53.204233+0000","last_became_peered":"2024-04-10T03:03:53.204233+0000","last_unstale":"2024-04-10T03:04:36.382482+0000","last_undegraded":"2024-04-10T03:04:36.382482+0000","last_fullsized":"2024-04-10T03:04:36.382482+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:05:24.600632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526936+0000","last_change":"2024-04-10T03:03:52.175336+0000","last_active":"2024-04-10T03:04:36.526936+0000","last_peered":"2024-04-10T03:04:36.526936+0000","last_clean":"2024-04-10T03:04:36.526936+0000","last_became_active":"2024-04-10T03:03:52.174912+0000","last_became_peered":"2024-04-10T03:03:52.174912+0000","last_unstale":"2024-04-10T03:04:36.526936+0000","last_undegraded":"2024-04-10T03:04:36.526936+0000","last_fullsized":"2024-04-10T03:04:36.526936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:40:46.315062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378970+0000","last_change":"2024-04-10T03:03:53.199493+0000","last_active":"2024-04-10T03:04:36.378970+0000","last_peered":"2024-04-10T03:04:36.378970+0000","last_clean":"2024-04-10T03:04:36.378970+0000","last_became_active":"2024-04-10T03:03:53.199197+0000","last_became_peered":"2024-04-10T03:03:53.199197+0000","last_unstale":"2024-04-10T03:04:36.378970+0000","last_undegraded":"2024-04-10T03:04:36.378970+0000","last_fullsized":"2024-04-10T03:04:36.378970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:48:23.646373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379104+0000","last_change":"2024-04-10T03:03:52.176719+0000","last_active":"2024-04-10T03:04:36.379104+0000","last_peered":"2024-04-10T03:04:36.379104+0000","last_clean":"2024-04-10T03:04:36.379104+0000","last_became_active":"2024-04-10T03:03:52.176341+0000","last_became_peered":"2024-04-10T03:03:52.176341+0000","last_unstale":"2024-04-10T03:04:36.379104+0000","last_undegraded":"2024-04-10T03:04:36.379104+0000","last_fullsized":"2024-04-10T03:04:36.379104+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:43.066934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377806+0000","last_change":"2024-04-10T03:03:53.193214+0000","last_active":"2024-04-10T03:04:36.377806+0000","last_peered":"2024-04-10T03:04:36.377806+0000","last_clean":"2024-04-10T03:04:36.377806+0000","last_became_active":"2024-04-10T03:03:53.192937+0000","last_became_peered":"2024-04-10T03:03:53.192937+0000","last_unstale":"2024-04-10T03:04:36.377806+0000","last_undegraded":"2024-04-10T03:04:36.377806+0000","last_fullsized":"2024-04-10T03:04:36.377806+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:26:42.349897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527079+0000","last_change":"2024-04-10T03:03:52.174299+0000","last_active":"2024-04-10T03:04:36.527079+0000","last_peered":"2024-04-10T03:04:36.527079+0000","last_clean":"2024-04-10T03:04:36.527079+0000","last_became_active":"2024-04-10T03:03:52.174019+0000","last_became_peered":"2024-04-10T03:03:52.174019+0000","last_unstale":"2024-04-10T03:04:36.527079+0000","last_undegraded":"2024-04-10T03:04:36.527079+0000","last_fullsized":"2024-04-10T03:04:36.527079+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:20:26.352214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381121+0000","last_change":"2024-04-10T03:03:53.201699+0000","last_active":"2024-04-10T03:04:36.381121+0000","last_peered":"2024-04-10T03:04:36.381121+0000","last_clean":"2024-04-10T03:04:36.381121+0000","last_became_active":"2024-04-10T03:03:53.201439+0000","last_became_peered":"2024-04-10T03:03:53.201439+0000","last_unstale":"2024-04-10T03:04:36.381121+0000","last_undegraded":"2024-04-10T03:04:36.381121+0000","last_fullsized":"2024-04-10T03:04:36.381121+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:30.816143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376771+0000","last_change":"2024-04-10T03:03:52.172115+0000","last_active":"2024-04-10T03:04:36.376771+0000","last_peered":"2024-04-10T03:04:36.376771+0000","last_clean":"2024-04-10T03:04:36.376771+0000","last_became_active":"2024-04-10T03:03:52.171799+0000","last_became_peered":"2024-04-10T03:03:52.171799+0000","last_unstale":"2024-04-10T03:04:36.376771+0000","last_undegraded":"2024-04-10T03:04:36.376771+0000","last_fullsized":"2024-04-10T03:04:36.376771+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:23:28.186006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067060+0000","last_change":"2024-04-10T03:04:38.067060+0000","last_active":"2024-04-10T03:04:38.067060+0000","last_peered":"2024-04-10T03:04:38.067060+0000","last_clean":"2024-04-10T03:04:38.067060+0000","last_became_active":"2024-04-10T03:04:38.066712+0000","last_became_peered":"2024-04-10T03:04:38.066712+0000","last_unstale":"2024-04-10T03:04:38.067060+0000","last_undegraded":"2024-04-10T03:04:38.067060+0000","last_fullsized":"2024-04-10T03:04:38.067060+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:40:30.850888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379295+0000","last_change":"2024-04-10T03:03:52.178805+0000","last_active":"2024-04-10T03:04:36.379295+0000","last_peered":"2024-04-10T03:04:36.379295+0000","last_clean":"2024-04-10T03:04:36.379295+0000","last_became_active":"2024-04-10T03:03:52.178486+0000","last_became_peered":"2024-04-10T03:03:52.178486+0000","last_unstale":"2024-04-10T03:04:36.379295+0000","last_undegraded":"2024-04-10T03:04:36.379295+0000","last_fullsized":"2024-04-10T03:04:36.379295+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:05:02.973862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383732+0000","last_change":"2024-04-10T03:03:53.199352+0000","last_active":"2024-04-10T03:04:36.383732+0000","last_peered":"2024-04-10T03:04:36.383732+0000","last_clean":"2024-04-10T03:04:36.383732+0000","last_became_active":"2024-04-10T03:03:53.199032+0000","last_became_peered":"2024-04-10T03:03:53.199032+0000","last_unstale":"2024-04-10T03:04:36.383732+0000","last_undegraded":"2024-04-10T03:04:36.383732+0000","last_fullsized":"2024-04-10T03:04:36.383732+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:02:50.425776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382415+0000","last_change":"2024-04-10T03:03:52.180915+0000","last_active":"2024-04-10T03:04:36.382415+0000","last_peered":"2024-04-10T03:04:36.382415+0000","last_clean":"2024-04-10T03:04:36.382415+0000","last_became_active":"2024-04-10T03:03:52.180518+0000","last_became_peered":"2024-04-10T03:03:52.180518+0000","last_unstale":"2024-04-10T03:04:36.382415+0000","last_undegraded":"2024-04-10T03:04:36.382415+0000","last_fullsized":"2024-04-10T03:04:36.382415+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:55:36.302101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379198+0000","last_change":"2024-04-10T03:03:53.204664+0000","last_active":"2024-04-10T03:04:36.379198+0000","last_peered":"2024-04-10T03:04:36.379198+0000","last_clean":"2024-04-10T03:04:36.379198+0000","last_became_active":"2024-04-10T03:03:53.204321+0000","last_became_peered":"2024-04-10T03:03:53.204321+0000","last_unstale":"2024-04-10T03:04:36.379198+0000","last_undegraded":"2024-04-10T03:04:36.379198+0000","last_fullsized":"2024-04-10T03:04:36.379198+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:13:16.538695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377721+0000","last_change":"2024-04-10T03:03:52.182973+0000","last_active":"2024-04-10T03:04:36.377721+0000","last_peered":"2024-04-10T03:04:36.377721+0000","last_clean":"2024-04-10T03:04:36.377721+0000","last_became_active":"2024-04-10T03:03:52.182697+0000","last_became_peered":"2024-04-10T03:03:52.182697+0000","last_unstale":"2024-04-10T03:04:36.377721+0000","last_undegraded":"2024-04-10T03:04:36.377721+0000","last_fullsized":"2024-04-10T03:04:36.377721+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:00:38.926901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381808+0000","last_change":"2024-04-10T03:03:53.206997+0000","last_active":"2024-04-10T03:04:36.381808+0000","last_peered":"2024-04-10T03:04:36.381808+0000","last_clean":"2024-04-10T03:04:36.381808+0000","last_became_active":"2024-04-10T03:03:53.206693+0000","last_became_peered":"2024-04-10T03:03:53.206693+0000","last_unstale":"2024-04-10T03:04:36.381808+0000","last_undegraded":"2024-04-10T03:04:36.381808+0000","last_fullsized":"2024-04-10T03:04:36.381808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:50:16.496041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378809+0000","last_change":"2024-04-10T03:03:52.172387+0000","last_active":"2024-04-10T03:04:36.378809+0000","last_peered":"2024-04-10T03:04:36.378809+0000","last_clean":"2024-04-10T03:04:36.378809+0000","last_became_active":"2024-04-10T03:03:52.172133+0000","last_became_peered":"2024-04-10T03:03:52.172133+0000","last_unstale":"2024-04-10T03:04:36.378809+0000","last_undegraded":"2024-04-10T03:04:36.378809+0000","last_fullsized":"2024-04-10T03:04:36.378809+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:15:29.496387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":905463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66400000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.496}]},{"osd":1,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.60299999999999998,"15min":0.56999999999999995},"min":{"1min":0.34200000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.4139999999999997,"5min":6.4139999999999997,"15min":6.4139999999999997},"last":0.432},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.53400000000000003,"15min":0.52800000000000002},"min":{"1min":0.38700000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.40899999999999997}]},{"osd":2,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.73599999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.58699999999999997,"15min":0.59199999999999997},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.69499999999999995,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.51200000000000001}]},{"osd":3,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.57799999999999996,"15min":0.58599999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68500000000000005,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.58799999999999997}]},{"osd":4,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.42499999999999999,"15min":0.41499999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.32000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56499999999999995}]},{"osd":5,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.46800000000000003,"15min":0.45400000000000001},"min":{"1min":0.36099999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.50700000000000001,"15min":0.495},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.65200000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.34899999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.54200000000000004,"15min":0.53400000000000003},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.63}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":905052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69199999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.65400000000000003}]},{"osd":1,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.52300000000000002,"15min":0.51300000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.628},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77700000000000002,"5min":1.012,"15min":1.012},"last":0.60499999999999998}]},{"osd":2,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76100000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.52900000000000003},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.71599999999999997}]},{"osd":3,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54100000000000004,"15min":0.53200000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.40000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.69899999999999995}]},{"osd":4,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45000000000000001,"15min":0.433},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.66000000000000003,"5min":0.752,"15min":0.752},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.46000000000000002,"15min":0.432},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.72099999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.72899999999999998}]},{"osd":5,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.45300000000000001,"15min":0.42299999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.75900000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.495,"15min":0.46899999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.755}]},{"osd":7,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.52200000000000002,"15min":0.51100000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.77100000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.315},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.376,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.8160000000000001,"5min":1.8160000000000001,"15min":1.8160000000000001},"last":0.501}]},{"osd":2,"last update":"Wed Apr 10 03:04:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.498,"5min":1.452,"15min":1.452},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52300000000000002,"5min":1.365,"15min":1.365},"last":0.54100000000000004}]},{"osd":3,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.44800000000000001,"15min":0.45400000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":1.395,"15min":1.395},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.44,"15min":0.433},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.55000000000000004,"5min":1.423,"15min":1.423},"last":0.58499999999999996}]},{"osd":4,"last update":"Wed Apr 10 03:03:58 2024","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.71299999999999997,"15min":0.60899999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":16.536999999999999,"5min":16.536999999999999,"15min":16.536999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73699999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.79400000000000004}]},{"osd":5,"last update":"Wed Apr 10 03:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76700000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78100000000000003,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.80800000000000005}]},{"osd":6,"last update":"Wed Apr 10 03:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.63400000000000001,"15min":0.60499999999999998},"min":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.61199999999999999,"15min":0.61099999999999999},"min":{"1min":0.50800000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.85199999999999998}]},{"osd":7,"last update":"Wed Apr 10 03:04:07 2024","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.76400000000000001,"15min":0.64700000000000002},"min":{"1min":0.44400000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":16.507999999999999,"5min":16.507999999999999,"15min":16.507999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70599999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.72999999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":904610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Apr 10 03:03:48 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.315},"min":{"1min":0.27000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.57999999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.36499999999999999,"15min":0.33600000000000002},"min":{"1min":0.26200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.54400000000000004,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.34000000000000002}]},{"osd":2,"last update":"Wed Apr 10 03:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40500000000000003,"15min":0.40200000000000002},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64700000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.317}]},{"osd":3,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.42199999999999999,"15min":0.432},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70299999999999996,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.30599999999999999}]},{"osd":4,"last update":"Wed Apr 10 03:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.441}]},{"osd":5,"last update":"Wed Apr 10 03:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.39700000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.91000000000000003,"5min":1.155,"15min":1.155},"last":0.46400000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.74299999999999999,"15min":0.70099999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0369999999999999,"5min":9.6449999999999996,"15min":9.6449999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.70899999999999996,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.022,"5min":9.6319999999999997,"15min":9.6319999999999997},"last":0.54600000000000004}]},{"osd":7,"last update":"Wed Apr 10 03:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.626,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.97499999999999998,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.42399999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.84399999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.59399999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35699999999999998,"15min":0.33700000000000002},"min":{"1min":0.22700000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.374,"15min":0.35399999999999998},"min":{"1min":0.28199999999999997,"5min":0.222,"15min":0.222},"max":{"1min":0.51500000000000001,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.34399999999999997}]},{"osd":1,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38600000000000001,"15min":0.36399999999999999},"min":{"1min":0.26500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.443},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.39900000000000002},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.47799999999999998}]},{"osd":3,"last update":"Wed Apr 10 03:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.45000000000000001,"15min":0.46200000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42999999999999999,"15min":0.44},"min":{"1min":0.307,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.38800000000000001}]},{"osd":4,"last update":"Wed Apr 10 03:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56599999999999995,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.64600000000000002,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.57899999999999996},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.66800000000000004}]},{"osd":5,"last update":"Wed Apr 10 03:04:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.55900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":19.402000000000001}]},{"osd":6,"last update":"Wed Apr 10 03:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.72299999999999998}]},{"osd":7,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57599999999999996,"15min":0.57299999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.51300000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":313740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.373,"15min":0.39700000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.39700000000000002,"15min":0.42299999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.51000000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.36399999999999999}]},{"osd":1,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.39100000000000001,"15min":0.42999999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.46999999999999997,"5min":0.752,"15min":0.752},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.43099999999999999,"15min":0.46100000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.52400000000000002,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.42499999999999999}]},{"osd":2,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.41199999999999998,"15min":0.44800000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.52100000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.45500000000000002,"15min":0.49199999999999999},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.50700000000000001,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.45900000000000002}]},{"osd":4,"last update":"Wed Apr 10 03:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71099999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.754},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57799999999999996,"15min":0.58799999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69999999999999996,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.78800000000000003}]},{"osd":5,"last update":"Wed Apr 10 03:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.39600000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.61799999999999999}]},{"osd":6,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81100000000000005,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.86899999999999999,"15min":0.998},"min":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.755,"5min":9.718,"15min":9.718},"last":0.68700000000000006}]},{"osd":7,"last update":"Wed Apr 10 03:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77300000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.012,"15min":1.0800000000000001},"min":{"1min":0.41799999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.84199999999999997,"5min":9.9160000000000004,"15min":9.9160000000000004},"last":0.71099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":737280,"data_stored":315083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51200000000000001,"15min":0.501},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.70699999999999996,"5min":0.745,"15min":0.745},"last":0.56100000000000005}]},{"osd":1,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.377},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.33100000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77000000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.32100000000000001}]},{"osd":2,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50800000000000001,"15min":0.5},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.38800000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.49199999999999999}]},{"osd":3,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.45000000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.69699999999999995,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Wed Apr 10 03:04:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.39900000000000002,"15min":0.38100000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.436,"15min":0.40999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.64600000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:04:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.52300000000000002,"15min":0.497},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.46400000000000002,"15min":0.45100000000000001},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52700000000000002}]},{"osd":7,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.63200000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757292,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":0.76800000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.42899999999999999}]},{"osd":1,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71399999999999997,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.58499999999999996,"15min":0.55800000000000005},"min":{"1min":0.41999999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":0.51000000000000001}]},{"osd":2,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.5270000000000001,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55100000000000005,"15min":0.55800000000000005},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73099999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.66700000000000004}]},{"osd":3,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.66700000000000004,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55100000000000005,"15min":0.54200000000000004},"min":{"1min":0.41599999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73999999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.61399999999999999}]},{"osd":4,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.437,"15min":0.42499999999999999},"min":{"1min":0.34200000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45900000000000002,"15min":0.44800000000000001},"min":{"1min":0.32400000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.76700000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.67600000000000005}]},{"osd":6,"last update":"Wed Apr 10 03:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.46100000000000002,"15min":0.44500000000000001},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.48799999999999999,"15min":0.46800000000000003},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64400000000000002}]},{"osd":7,"last update":"Wed Apr 10 03:04:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70599999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78500000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.69699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:04:50.621 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:04:50.873 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:51.690 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[29301]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:51.690 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[29301]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:51.690 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:51.690 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[39651]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:51.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[39651]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:51.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:51 smithi152 ceph-mon[33523]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:51 smithi152 ceph-mon[33523]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:51.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:04:52.714 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:52.714 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:52] "GET /metrics HTTP/1.1" 200 239133 "" "Prometheus/2.43.0" 2024-04-10T03:04:52.718 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:04:53.435 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2024-04-10T03:04:52.102363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386276,"kb_used_data":7420,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574940,"statfs":{"total":767960285184,"available":765516738560,"internally_reserved":0,"allocated":7598080,"data_stored":4286142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002627"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378662+0000","last_change":"2024-04-10T03:03:53.199403+0000","last_active":"2024-04-10T03:04:36.378662+0000","last_peered":"2024-04-10T03:04:36.378662+0000","last_clean":"2024-04-10T03:04:36.378662+0000","last_became_active":"2024-04-10T03:03:53.199091+0000","last_became_peered":"2024-04-10T03:03:53.199091+0000","last_unstale":"2024-04-10T03:04:36.378662+0000","last_undegraded":"2024-04-10T03:04:36.378662+0000","last_fullsized":"2024-04-10T03:04:36.378662+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:07:04.347723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379215+0000","last_change":"2024-04-10T03:03:52.174045+0000","last_active":"2024-04-10T03:04:36.379215+0000","last_peered":"2024-04-10T03:04:36.379215+0000","last_clean":"2024-04-10T03:04:36.379215+0000","last_became_active":"2024-04-10T03:03:52.173281+0000","last_became_peered":"2024-04-10T03:03:52.173281+0000","last_unstale":"2024-04-10T03:04:36.379215+0000","last_undegraded":"2024-04-10T03:04:36.379215+0000","last_fullsized":"2024-04-10T03:04:36.379215+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:18:50.887661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381047+0000","last_change":"2024-04-10T03:03:53.196289+0000","last_active":"2024-04-10T03:04:36.381047+0000","last_peered":"2024-04-10T03:04:36.381047+0000","last_clean":"2024-04-10T03:04:36.381047+0000","last_became_active":"2024-04-10T03:03:53.196012+0000","last_became_peered":"2024-04-10T03:03:53.196012+0000","last_unstale":"2024-04-10T03:04:36.381047+0000","last_undegraded":"2024-04-10T03:04:36.381047+0000","last_fullsized":"2024-04-10T03:04:36.381047+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:46:28.369446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382107+0000","last_change":"2024-04-10T03:03:52.175141+0000","last_active":"2024-04-10T03:04:36.382107+0000","last_peered":"2024-04-10T03:04:36.382107+0000","last_clean":"2024-04-10T03:04:36.382107+0000","last_became_active":"2024-04-10T03:03:52.174710+0000","last_became_peered":"2024-04-10T03:03:52.174710+0000","last_unstale":"2024-04-10T03:04:36.382107+0000","last_undegraded":"2024-04-10T03:04:36.382107+0000","last_fullsized":"2024-04-10T03:04:36.382107+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:08:58.484193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378930+0000","last_change":"2024-04-10T03:03:53.197126+0000","last_active":"2024-04-10T03:04:36.378930+0000","last_peered":"2024-04-10T03:04:36.378930+0000","last_clean":"2024-04-10T03:04:36.378930+0000","last_became_active":"2024-04-10T03:03:53.196791+0000","last_became_peered":"2024-04-10T03:03:53.196791+0000","last_unstale":"2024-04-10T03:04:36.378930+0000","last_undegraded":"2024-04-10T03:04:36.378930+0000","last_fullsized":"2024-04-10T03:04:36.378930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:34:30.736007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377011+0000","last_change":"2024-04-10T03:03:52.169154+0000","last_active":"2024-04-10T03:04:36.377011+0000","last_peered":"2024-04-10T03:04:36.377011+0000","last_clean":"2024-04-10T03:04:36.377011+0000","last_became_active":"2024-04-10T03:03:52.168854+0000","last_became_peered":"2024-04-10T03:03:52.168854+0000","last_unstale":"2024-04-10T03:04:36.377011+0000","last_undegraded":"2024-04-10T03:04:36.377011+0000","last_fullsized":"2024-04-10T03:04:36.377011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:01:21.091245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379198+0000","last_change":"2024-04-10T03:03:53.200560+0000","last_active":"2024-04-10T03:04:36.379198+0000","last_peered":"2024-04-10T03:04:36.379198+0000","last_clean":"2024-04-10T03:04:36.379198+0000","last_became_active":"2024-04-10T03:03:53.200217+0000","last_became_peered":"2024-04-10T03:03:53.200217+0000","last_unstale":"2024-04-10T03:04:36.379198+0000","last_undegraded":"2024-04-10T03:04:36.379198+0000","last_fullsized":"2024-04-10T03:04:36.379198+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:19:22.060465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376896+0000","last_change":"2024-04-10T03:03:52.170933+0000","last_active":"2024-04-10T03:04:36.376896+0000","last_peered":"2024-04-10T03:04:36.376896+0000","last_clean":"2024-04-10T03:04:36.376896+0000","last_became_active":"2024-04-10T03:03:52.170616+0000","last_became_peered":"2024-04-10T03:03:52.170616+0000","last_unstale":"2024-04-10T03:04:36.376896+0000","last_undegraded":"2024-04-10T03:04:36.376896+0000","last_fullsized":"2024-04-10T03:04:36.376896+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:29.189824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376917+0000","last_change":"2024-04-10T03:03:53.193547+0000","last_active":"2024-04-10T03:04:36.376917+0000","last_peered":"2024-04-10T03:04:36.376917+0000","last_clean":"2024-04-10T03:04:36.376917+0000","last_became_active":"2024-04-10T03:03:53.193249+0000","last_became_peered":"2024-04-10T03:03:53.193249+0000","last_unstale":"2024-04-10T03:04:36.376917+0000","last_undegraded":"2024-04-10T03:04:36.376917+0000","last_fullsized":"2024-04-10T03:04:36.376917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:17:58.101131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379041+0000","last_change":"2024-04-10T03:03:52.175545+0000","last_active":"2024-04-10T03:04:36.379041+0000","last_peered":"2024-04-10T03:04:36.379041+0000","last_clean":"2024-04-10T03:04:36.379041+0000","last_became_active":"2024-04-10T03:03:52.175288+0000","last_became_peered":"2024-04-10T03:03:52.175288+0000","last_unstale":"2024-04-10T03:04:36.379041+0000","last_undegraded":"2024-04-10T03:04:36.379041+0000","last_fullsized":"2024-04-10T03:04:36.379041+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:45:19.900779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379043+0000","last_change":"2024-04-10T03:03:53.193384+0000","last_active":"2024-04-10T03:04:36.379043+0000","last_peered":"2024-04-10T03:04:36.379043+0000","last_clean":"2024-04-10T03:04:36.379043+0000","last_became_active":"2024-04-10T03:03:53.193048+0000","last_became_peered":"2024-04-10T03:03:53.193048+0000","last_unstale":"2024-04-10T03:04:36.379043+0000","last_undegraded":"2024-04-10T03:04:36.379043+0000","last_fullsized":"2024-04-10T03:04:36.379043+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:10:41.808720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379274+0000","last_change":"2024-04-10T03:03:52.171292+0000","last_active":"2024-04-10T03:04:36.379274+0000","last_peered":"2024-04-10T03:04:36.379274+0000","last_clean":"2024-04-10T03:04:36.379274+0000","last_became_active":"2024-04-10T03:03:52.171030+0000","last_became_peered":"2024-04-10T03:03:52.171030+0000","last_unstale":"2024-04-10T03:04:36.379274+0000","last_undegraded":"2024-04-10T03:04:36.379274+0000","last_fullsized":"2024-04-10T03:04:36.379274+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:10:28.134300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383763+0000","last_change":"2024-04-10T03:02:38.143240+0000","last_active":"2024-04-10T03:04:36.383763+0000","last_peered":"2024-04-10T03:04:36.383763+0000","last_clean":"2024-04-10T03:04:36.383763+0000","last_became_active":"2024-04-10T03:02:38.142941+0000","last_became_peered":"2024-04-10T03:02:38.142941+0000","last_unstale":"2024-04-10T03:04:36.383763+0000","last_undegraded":"2024-04-10T03:04:36.383763+0000","last_fullsized":"2024-04-10T03:04:36.383763+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:25:57.656837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382234+0000","last_change":"2024-04-10T03:03:53.201926+0000","last_active":"2024-04-10T03:04:36.382234+0000","last_peered":"2024-04-10T03:04:36.382234+0000","last_clean":"2024-04-10T03:04:36.382234+0000","last_became_active":"2024-04-10T03:03:53.201614+0000","last_became_peered":"2024-04-10T03:03:53.201614+0000","last_unstale":"2024-04-10T03:04:36.382234+0000","last_undegraded":"2024-04-10T03:04:36.382234+0000","last_fullsized":"2024-04-10T03:04:36.382234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:06:56.966117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379568+0000","last_change":"2024-04-10T03:02:38.321430+0000","last_active":"2024-04-10T03:04:36.379568+0000","last_peered":"2024-04-10T03:04:36.379568+0000","last_clean":"2024-04-10T03:04:36.379568+0000","last_became_active":"2024-04-10T03:02:38.321079+0000","last_became_peered":"2024-04-10T03:02:38.321079+0000","last_unstale":"2024-04-10T03:04:36.379568+0000","last_undegraded":"2024-04-10T03:04:36.379568+0000","last_fullsized":"2024-04-10T03:04:36.379568+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:39.342473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382310+0000","last_change":"2024-04-10T03:03:52.182258+0000","last_active":"2024-04-10T03:04:36.382310+0000","last_peered":"2024-04-10T03:04:36.382310+0000","last_clean":"2024-04-10T03:04:36.382310+0000","last_became_active":"2024-04-10T03:03:52.181986+0000","last_became_peered":"2024-04-10T03:03:52.181986+0000","last_unstale":"2024-04-10T03:04:36.382310+0000","last_undegraded":"2024-04-10T03:04:36.382310+0000","last_fullsized":"2024-04-10T03:04:36.382310+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:32:34.330120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378909+0000","last_change":"2024-04-10T03:03:53.205741+0000","last_active":"2024-04-10T03:04:36.378909+0000","last_peered":"2024-04-10T03:04:36.378909+0000","last_clean":"2024-04-10T03:04:36.378909+0000","last_became_active":"2024-04-10T03:03:53.205418+0000","last_became_peered":"2024-04-10T03:03:53.205418+0000","last_unstale":"2024-04-10T03:04:36.378909+0000","last_undegraded":"2024-04-10T03:04:36.378909+0000","last_fullsized":"2024-04-10T03:04:36.378909+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:37:35.498166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378646+0000","last_change":"2024-04-10T03:03:52.176007+0000","last_active":"2024-04-10T03:04:36.378646+0000","last_peered":"2024-04-10T03:04:36.378646+0000","last_clean":"2024-04-10T03:04:36.378646+0000","last_became_active":"2024-04-10T03:03:52.175722+0000","last_became_peered":"2024-04-10T03:03:52.175722+0000","last_unstale":"2024-04-10T03:04:36.378646+0000","last_undegraded":"2024-04-10T03:04:36.378646+0000","last_fullsized":"2024-04-10T03:04:36.378646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:48.316168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527086+0000","last_change":"2024-04-10T03:02:38.321148+0000","last_active":"2024-04-10T03:04:36.527086+0000","last_peered":"2024-04-10T03:04:36.527086+0000","last_clean":"2024-04-10T03:04:36.527086+0000","last_became_active":"2024-04-10T03:02:38.320773+0000","last_became_peered":"2024-04-10T03:02:38.320773+0000","last_unstale":"2024-04-10T03:04:36.527086+0000","last_undegraded":"2024-04-10T03:04:36.527086+0000","last_fullsized":"2024-04-10T03:04:36.527086+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:18:30.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067157+0000","last_change":"2024-04-10T03:04:38.067157+0000","last_active":"2024-04-10T03:04:38.067157+0000","last_peered":"2024-04-10T03:04:38.067157+0000","last_clean":"2024-04-10T03:04:38.067157+0000","last_became_active":"2024-04-10T03:04:38.066711+0000","last_became_peered":"2024-04-10T03:04:38.066711+0000","last_unstale":"2024-04-10T03:04:38.067157+0000","last_undegraded":"2024-04-10T03:04:38.067157+0000","last_fullsized":"2024-04-10T03:04:38.067157+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:14:54.573143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378706+0000","last_change":"2024-04-10T03:03:52.177504+0000","last_active":"2024-04-10T03:04:36.378706+0000","last_peered":"2024-04-10T03:04:36.378706+0000","last_clean":"2024-04-10T03:04:36.378706+0000","last_became_active":"2024-04-10T03:03:52.177241+0000","last_became_peered":"2024-04-10T03:03:52.177241+0000","last_unstale":"2024-04-10T03:04:36.378706+0000","last_undegraded":"2024-04-10T03:04:36.378706+0000","last_fullsized":"2024-04-10T03:04:36.378706+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:21:52.152273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526782+0000","last_change":"2024-04-10T03:02:38.141165+0000","last_active":"2024-04-10T03:04:36.526782+0000","last_peered":"2024-04-10T03:04:36.526782+0000","last_clean":"2024-04-10T03:04:36.526782+0000","last_became_active":"2024-04-10T03:02:38.140854+0000","last_became_peered":"2024-04-10T03:02:38.140854+0000","last_unstale":"2024-04-10T03:04:36.526782+0000","last_undegraded":"2024-04-10T03:04:36.526782+0000","last_fullsized":"2024-04-10T03:04:36.526782+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:29:15.802525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383683+0000","last_change":"2024-04-10T03:03:53.201722+0000","last_active":"2024-04-10T03:04:36.383683+0000","last_peered":"2024-04-10T03:04:36.383683+0000","last_clean":"2024-04-10T03:04:36.383683+0000","last_became_active":"2024-04-10T03:03:53.201394+0000","last_became_peered":"2024-04-10T03:03:53.201394+0000","last_unstale":"2024-04-10T03:04:36.383683+0000","last_undegraded":"2024-04-10T03:04:36.383683+0000","last_fullsized":"2024-04-10T03:04:36.383683+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:16.107170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381359+0000","last_change":"2024-04-10T03:03:52.177805+0000","last_active":"2024-04-10T03:04:36.381359+0000","last_peered":"2024-04-10T03:04:36.381359+0000","last_clean":"2024-04-10T03:04:36.381359+0000","last_became_active":"2024-04-10T03:03:52.174720+0000","last_became_peered":"2024-04-10T03:03:52.174720+0000","last_unstale":"2024-04-10T03:04:36.381359+0000","last_undegraded":"2024-04-10T03:04:36.381359+0000","last_fullsized":"2024-04-10T03:04:36.381359+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:09:38.594358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526786+0000","last_change":"2024-04-10T03:02:08.306776+0000","last_active":"2024-04-10T03:04:36.526786+0000","last_peered":"2024-04-10T03:04:36.526786+0000","last_clean":"2024-04-10T03:04:36.526786+0000","last_became_active":"2024-04-10T03:02:08.306446+0000","last_became_peered":"2024-04-10T03:02:08.306446+0000","last_unstale":"2024-04-10T03:04:36.526786+0000","last_undegraded":"2024-04-10T03:04:36.526786+0000","last_fullsized":"2024-04-10T03:04:36.526786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:40:01.838564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376787+0000","last_change":"2024-04-10T03:02:38.143477+0000","last_active":"2024-04-10T03:04:36.376787+0000","last_peered":"2024-04-10T03:04:36.376787+0000","last_clean":"2024-04-10T03:04:36.376787+0000","last_became_active":"2024-04-10T03:02:38.143077+0000","last_became_peered":"2024-04-10T03:02:38.143077+0000","last_unstale":"2024-04-10T03:04:36.376787+0000","last_undegraded":"2024-04-10T03:04:36.376787+0000","last_fullsized":"2024-04-10T03:04:36.376787+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:02:03.974803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379545+0000","last_change":"2024-04-10T03:03:53.196065+0000","last_active":"2024-04-10T03:04:36.379545+0000","last_peered":"2024-04-10T03:04:36.379545+0000","last_clean":"2024-04-10T03:04:36.379545+0000","last_became_active":"2024-04-10T03:03:53.195767+0000","last_became_peered":"2024-04-10T03:03:53.195767+0000","last_unstale":"2024-04-10T03:04:36.379545+0000","last_undegraded":"2024-04-10T03:04:36.379545+0000","last_fullsized":"2024-04-10T03:04:36.379545+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:03:51.593233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377578+0000","last_change":"2024-04-10T03:03:52.171011+0000","last_active":"2024-04-10T03:04:36.377578+0000","last_peered":"2024-04-10T03:04:36.377578+0000","last_clean":"2024-04-10T03:04:36.377578+0000","last_became_active":"2024-04-10T03:03:52.170746+0000","last_became_peered":"2024-04-10T03:03:52.170746+0000","last_unstale":"2024-04-10T03:04:36.377578+0000","last_undegraded":"2024-04-10T03:04:36.377578+0000","last_fullsized":"2024-04-10T03:04:36.377578+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:39:06.035417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376736+0000","last_change":"2024-04-10T03:02:38.552665+0000","last_active":"2024-04-10T03:04:36.376736+0000","last_peered":"2024-04-10T03:04:36.376736+0000","last_clean":"2024-04-10T03:04:36.376736+0000","last_became_active":"2024-04-10T03:02:38.552335+0000","last_became_peered":"2024-04-10T03:02:38.552335+0000","last_unstale":"2024-04-10T03:04:36.376736+0000","last_undegraded":"2024-04-10T03:04:36.376736+0000","last_fullsized":"2024-04-10T03:04:36.376736+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:54:39.380508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378571+0000","last_change":"2024-04-10T03:03:53.200043+0000","last_active":"2024-04-10T03:04:36.378571+0000","last_peered":"2024-04-10T03:04:36.378571+0000","last_clean":"2024-04-10T03:04:36.378571+0000","last_became_active":"2024-04-10T03:03:53.199780+0000","last_became_peered":"2024-04-10T03:03:53.199780+0000","last_unstale":"2024-04-10T03:04:36.378571+0000","last_undegraded":"2024-04-10T03:04:36.378571+0000","last_fullsized":"2024-04-10T03:04:36.378571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:01:12.981096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379127+0000","last_change":"2024-04-10T03:03:52.176230+0000","last_active":"2024-04-10T03:04:36.379127+0000","last_peered":"2024-04-10T03:04:36.379127+0000","last_clean":"2024-04-10T03:04:36.379127+0000","last_became_active":"2024-04-10T03:03:52.175660+0000","last_became_peered":"2024-04-10T03:03:52.175660+0000","last_unstale":"2024-04-10T03:04:36.379127+0000","last_undegraded":"2024-04-10T03:04:36.379127+0000","last_fullsized":"2024-04-10T03:04:36.379127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:41:32.982610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376661+0000","last_change":"2024-04-10T03:02:38.139183+0000","last_active":"2024-04-10T03:04:36.376661+0000","last_peered":"2024-04-10T03:04:36.376661+0000","last_clean":"2024-04-10T03:04:36.376661+0000","last_became_active":"2024-04-10T03:02:38.138885+0000","last_became_peered":"2024-04-10T03:02:38.138885+0000","last_unstale":"2024-04-10T03:04:36.376661+0000","last_undegraded":"2024-04-10T03:04:36.376661+0000","last_fullsized":"2024-04-10T03:04:36.376661+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:52:28.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377886+0000","last_change":"2024-04-10T03:03:53.198290+0000","last_active":"2024-04-10T03:04:36.377886+0000","last_peered":"2024-04-10T03:04:36.377886+0000","last_clean":"2024-04-10T03:04:36.377886+0000","last_became_active":"2024-04-10T03:03:53.198009+0000","last_became_peered":"2024-04-10T03:03:53.198009+0000","last_unstale":"2024-04-10T03:04:36.377886+0000","last_undegraded":"2024-04-10T03:04:36.377886+0000","last_fullsized":"2024-04-10T03:04:36.377886+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:50:00.001489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376897+0000","last_change":"2024-04-10T03:03:52.173812+0000","last_active":"2024-04-10T03:04:36.376897+0000","last_peered":"2024-04-10T03:04:36.376897+0000","last_clean":"2024-04-10T03:04:36.376897+0000","last_became_active":"2024-04-10T03:03:52.173468+0000","last_became_peered":"2024-04-10T03:03:52.173468+0000","last_unstale":"2024-04-10T03:04:36.376897+0000","last_undegraded":"2024-04-10T03:04:36.376897+0000","last_fullsized":"2024-04-10T03:04:36.376897+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:08:33.209359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378503+0000","last_change":"2024-04-10T03:02:38.140035+0000","last_active":"2024-04-10T03:04:36.378503+0000","last_peered":"2024-04-10T03:04:36.378503+0000","last_clean":"2024-04-10T03:04:36.378503+0000","last_became_active":"2024-04-10T03:02:38.139760+0000","last_became_peered":"2024-04-10T03:02:38.139760+0000","last_unstale":"2024-04-10T03:04:36.378503+0000","last_undegraded":"2024-04-10T03:04:36.378503+0000","last_fullsized":"2024-04-10T03:04:36.378503+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:31:22.147412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379469+0000","last_change":"2024-04-10T03:03:53.200087+0000","last_active":"2024-04-10T03:04:36.379469+0000","last_peered":"2024-04-10T03:04:36.379469+0000","last_clean":"2024-04-10T03:04:36.379469+0000","last_became_active":"2024-04-10T03:03:53.199771+0000","last_became_peered":"2024-04-10T03:03:53.199771+0000","last_unstale":"2024-04-10T03:04:36.379469+0000","last_undegraded":"2024-04-10T03:04:36.379469+0000","last_fullsized":"2024-04-10T03:04:36.379469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:10:48.839174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382316+0000","last_change":"2024-04-10T03:03:52.178981+0000","last_active":"2024-04-10T03:04:36.382316+0000","last_peered":"2024-04-10T03:04:36.382316+0000","last_clean":"2024-04-10T03:04:36.382316+0000","last_became_active":"2024-04-10T03:03:52.178210+0000","last_became_peered":"2024-04-10T03:03:52.178210+0000","last_unstale":"2024-04-10T03:04:36.382316+0000","last_undegraded":"2024-04-10T03:04:36.382316+0000","last_fullsized":"2024-04-10T03:04:36.382316+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:07:53.916420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379055+0000","last_change":"2024-04-10T03:03:53.200095+0000","last_active":"2024-04-10T03:04:36.379055+0000","last_peered":"2024-04-10T03:04:36.379055+0000","last_clean":"2024-04-10T03:04:36.379055+0000","last_became_active":"2024-04-10T03:03:53.199779+0000","last_became_peered":"2024-04-10T03:03:53.199779+0000","last_unstale":"2024-04-10T03:04:36.379055+0000","last_undegraded":"2024-04-10T03:04:36.379055+0000","last_fullsized":"2024-04-10T03:04:36.379055+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:16:44.877622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378771+0000","last_change":"2024-04-10T03:03:52.172894+0000","last_active":"2024-04-10T03:04:36.378771+0000","last_peered":"2024-04-10T03:04:36.378771+0000","last_clean":"2024-04-10T03:04:36.378771+0000","last_became_active":"2024-04-10T03:03:52.172669+0000","last_became_peered":"2024-04-10T03:03:52.172669+0000","last_unstale":"2024-04-10T03:04:36.378771+0000","last_undegraded":"2024-04-10T03:04:36.378771+0000","last_fullsized":"2024-04-10T03:04:36.378771+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:06:44.554916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067113+0000","last_change":"2024-04-10T03:04:38.067113+0000","last_active":"2024-04-10T03:04:38.067113+0000","last_peered":"2024-04-10T03:04:38.067113+0000","last_clean":"2024-04-10T03:04:38.067113+0000","last_became_active":"2024-04-10T03:04:38.066631+0000","last_became_peered":"2024-04-10T03:04:38.066631+0000","last_unstale":"2024-04-10T03:04:38.067113+0000","last_undegraded":"2024-04-10T03:04:38.067113+0000","last_fullsized":"2024-04-10T03:04:38.067113+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:37:07.088775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383497+0000","last_change":"2024-04-10T03:03:52.176489+0000","last_active":"2024-04-10T03:04:36.383497+0000","last_peered":"2024-04-10T03:04:36.383497+0000","last_clean":"2024-04-10T03:04:36.383497+0000","last_became_active":"2024-04-10T03:03:52.176198+0000","last_became_peered":"2024-04-10T03:03:52.176198+0000","last_unstale":"2024-04-10T03:04:36.383497+0000","last_undegraded":"2024-04-10T03:04:36.383497+0000","last_fullsized":"2024-04-10T03:04:36.383497+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:35:37.911620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378905+0000","last_change":"2024-04-10T03:03:53.197184+0000","last_active":"2024-04-10T03:04:36.378905+0000","last_peered":"2024-04-10T03:04:36.378905+0000","last_clean":"2024-04-10T03:04:36.378905+0000","last_became_active":"2024-04-10T03:03:53.196842+0000","last_became_peered":"2024-04-10T03:03:53.196842+0000","last_unstale":"2024-04-10T03:04:36.378905+0000","last_undegraded":"2024-04-10T03:04:36.378905+0000","last_fullsized":"2024-04-10T03:04:36.378905+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:42:29.861897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381519+0000","last_change":"2024-04-10T03:03:52.181581+0000","last_active":"2024-04-10T03:04:36.381519+0000","last_peered":"2024-04-10T03:04:36.381519+0000","last_clean":"2024-04-10T03:04:36.381519+0000","last_became_active":"2024-04-10T03:03:52.181307+0000","last_became_peered":"2024-04-10T03:03:52.181307+0000","last_unstale":"2024-04-10T03:04:36.381519+0000","last_undegraded":"2024-04-10T03:04:36.381519+0000","last_fullsized":"2024-04-10T03:04:36.381519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:39:43.513327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379350+0000","last_change":"2024-04-10T03:03:53.196706+0000","last_active":"2024-04-10T03:04:36.379350+0000","last_peered":"2024-04-10T03:04:36.379350+0000","last_clean":"2024-04-10T03:04:36.379350+0000","last_became_active":"2024-04-10T03:03:53.196450+0000","last_became_peered":"2024-04-10T03:03:53.196450+0000","last_unstale":"2024-04-10T03:04:36.379350+0000","last_undegraded":"2024-04-10T03:04:36.379350+0000","last_fullsized":"2024-04-10T03:04:36.379350+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:38:56.781064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376821+0000","last_change":"2024-04-10T03:03:52.175356+0000","last_active":"2024-04-10T03:04:36.376821+0000","last_peered":"2024-04-10T03:04:36.376821+0000","last_clean":"2024-04-10T03:04:36.376821+0000","last_became_active":"2024-04-10T03:03:52.175063+0000","last_became_peered":"2024-04-10T03:03:52.175063+0000","last_unstale":"2024-04-10T03:04:36.376821+0000","last_undegraded":"2024-04-10T03:04:36.376821+0000","last_fullsized":"2024-04-10T03:04:36.376821+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:13:07.527860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383436+0000","last_change":"2024-04-10T03:03:53.200545+0000","last_active":"2024-04-10T03:04:36.383436+0000","last_peered":"2024-04-10T03:04:36.383436+0000","last_clean":"2024-04-10T03:04:36.383436+0000","last_became_active":"2024-04-10T03:03:53.200173+0000","last_became_peered":"2024-04-10T03:03:53.200173+0000","last_unstale":"2024-04-10T03:04:36.383436+0000","last_undegraded":"2024-04-10T03:04:36.383436+0000","last_fullsized":"2024-04-10T03:04:36.383436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:53:35.906724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527172+0000","last_change":"2024-04-10T03:03:52.170665+0000","last_active":"2024-04-10T03:04:36.527172+0000","last_peered":"2024-04-10T03:04:36.527172+0000","last_clean":"2024-04-10T03:04:36.527172+0000","last_became_active":"2024-04-10T03:03:52.170385+0000","last_became_peered":"2024-04-10T03:03:52.170385+0000","last_unstale":"2024-04-10T03:04:36.527172+0000","last_undegraded":"2024-04-10T03:04:36.527172+0000","last_fullsized":"2024-04-10T03:04:36.527172+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:36:28.794484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378728+0000","last_change":"2024-04-10T03:03:53.199941+0000","last_active":"2024-04-10T03:04:36.378728+0000","last_peered":"2024-04-10T03:04:36.378728+0000","last_clean":"2024-04-10T03:04:36.378728+0000","last_became_active":"2024-04-10T03:03:53.199621+0000","last_became_peered":"2024-04-10T03:03:53.199621+0000","last_unstale":"2024-04-10T03:04:36.378728+0000","last_undegraded":"2024-04-10T03:04:36.378728+0000","last_fullsized":"2024-04-10T03:04:36.378728+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:24:25.332895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527115+0000","last_change":"2024-04-10T03:03:52.176385+0000","last_active":"2024-04-10T03:04:36.527115+0000","last_peered":"2024-04-10T03:04:36.527115+0000","last_clean":"2024-04-10T03:04:36.527115+0000","last_became_active":"2024-04-10T03:03:52.176122+0000","last_became_peered":"2024-04-10T03:03:52.176122+0000","last_unstale":"2024-04-10T03:04:36.527115+0000","last_undegraded":"2024-04-10T03:04:36.527115+0000","last_fullsized":"2024-04-10T03:04:36.527115+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:15:20.134402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376652+0000","last_change":"2024-04-10T03:03:53.200529+0000","last_active":"2024-04-10T03:04:36.376652+0000","last_peered":"2024-04-10T03:04:36.376652+0000","last_clean":"2024-04-10T03:04:36.376652+0000","last_became_active":"2024-04-10T03:03:53.199080+0000","last_became_peered":"2024-04-10T03:03:53.199080+0000","last_unstale":"2024-04-10T03:04:36.376652+0000","last_undegraded":"2024-04-10T03:04:36.376652+0000","last_fullsized":"2024-04-10T03:04:36.376652+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T15:00:49.737785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527202+0000","last_change":"2024-04-10T03:03:52.178341+0000","last_active":"2024-04-10T03:04:36.527202+0000","last_peered":"2024-04-10T03:04:36.527202+0000","last_clean":"2024-04-10T03:04:36.527202+0000","last_became_active":"2024-04-10T03:03:52.178082+0000","last_became_peered":"2024-04-10T03:03:52.178082+0000","last_unstale":"2024-04-10T03:04:36.527202+0000","last_undegraded":"2024-04-10T03:04:36.527202+0000","last_fullsized":"2024-04-10T03:04:36.527202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:18:59.101964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382386+0000","last_change":"2024-04-10T03:03:53.205118+0000","last_active":"2024-04-10T03:04:36.382386+0000","last_peered":"2024-04-10T03:04:36.382386+0000","last_clean":"2024-04-10T03:04:36.382386+0000","last_became_active":"2024-04-10T03:03:53.204845+0000","last_became_peered":"2024-04-10T03:03:53.204845+0000","last_unstale":"2024-04-10T03:04:36.382386+0000","last_undegraded":"2024-04-10T03:04:36.382386+0000","last_fullsized":"2024-04-10T03:04:36.382386+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:23:30.856777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383866+0000","last_change":"2024-04-10T03:03:52.174189+0000","last_active":"2024-04-10T03:04:36.383866+0000","last_peered":"2024-04-10T03:04:36.383866+0000","last_clean":"2024-04-10T03:04:36.383866+0000","last_became_active":"2024-04-10T03:03:52.173057+0000","last_became_peered":"2024-04-10T03:03:52.173057+0000","last_unstale":"2024-04-10T03:04:36.383866+0000","last_undegraded":"2024-04-10T03:04:36.383866+0000","last_fullsized":"2024-04-10T03:04:36.383866+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:30:45.451822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379127+0000","last_change":"2024-04-10T03:03:53.200354+0000","last_active":"2024-04-10T03:04:36.379127+0000","last_peered":"2024-04-10T03:04:36.379127+0000","last_clean":"2024-04-10T03:04:36.379127+0000","last_became_active":"2024-04-10T03:03:53.200095+0000","last_became_peered":"2024-04-10T03:03:53.200095+0000","last_unstale":"2024-04-10T03:04:36.379127+0000","last_undegraded":"2024-04-10T03:04:36.379127+0000","last_fullsized":"2024-04-10T03:04:36.379127+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:33:26.829187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526814+0000","last_change":"2024-04-10T03:03:52.171594+0000","last_active":"2024-04-10T03:04:36.526814+0000","last_peered":"2024-04-10T03:04:36.526814+0000","last_clean":"2024-04-10T03:04:36.526814+0000","last_became_active":"2024-04-10T03:03:52.171314+0000","last_became_peered":"2024-04-10T03:03:52.171314+0000","last_unstale":"2024-04-10T03:04:36.526814+0000","last_undegraded":"2024-04-10T03:04:36.526814+0000","last_fullsized":"2024-04-10T03:04:36.526814+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:37:55.120481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376892+0000","last_change":"2024-04-10T03:03:53.201997+0000","last_active":"2024-04-10T03:04:36.376892+0000","last_peered":"2024-04-10T03:04:36.376892+0000","last_clean":"2024-04-10T03:04:36.376892+0000","last_became_active":"2024-04-10T03:03:53.201679+0000","last_became_peered":"2024-04-10T03:03:53.201679+0000","last_unstale":"2024-04-10T03:04:36.376892+0000","last_undegraded":"2024-04-10T03:04:36.376892+0000","last_fullsized":"2024-04-10T03:04:36.376892+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:26:50.320247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379350+0000","last_change":"2024-04-10T03:03:52.179168+0000","last_active":"2024-04-10T03:04:36.379350+0000","last_peered":"2024-04-10T03:04:36.379350+0000","last_clean":"2024-04-10T03:04:36.379350+0000","last_became_active":"2024-04-10T03:03:52.178906+0000","last_became_peered":"2024-04-10T03:03:52.178906+0000","last_unstale":"2024-04-10T03:04:36.379350+0000","last_undegraded":"2024-04-10T03:04:36.379350+0000","last_fullsized":"2024-04-10T03:04:36.379350+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:33:10.623935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382482+0000","last_change":"2024-04-10T03:03:53.204493+0000","last_active":"2024-04-10T03:04:36.382482+0000","last_peered":"2024-04-10T03:04:36.382482+0000","last_clean":"2024-04-10T03:04:36.382482+0000","last_became_active":"2024-04-10T03:03:53.204233+0000","last_became_peered":"2024-04-10T03:03:53.204233+0000","last_unstale":"2024-04-10T03:04:36.382482+0000","last_undegraded":"2024-04-10T03:04:36.382482+0000","last_fullsized":"2024-04-10T03:04:36.382482+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:05:24.600632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.526936+0000","last_change":"2024-04-10T03:03:52.175336+0000","last_active":"2024-04-10T03:04:36.526936+0000","last_peered":"2024-04-10T03:04:36.526936+0000","last_clean":"2024-04-10T03:04:36.526936+0000","last_became_active":"2024-04-10T03:03:52.174912+0000","last_became_peered":"2024-04-10T03:03:52.174912+0000","last_unstale":"2024-04-10T03:04:36.526936+0000","last_undegraded":"2024-04-10T03:04:36.526936+0000","last_fullsized":"2024-04-10T03:04:36.526936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:40:46.315062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378970+0000","last_change":"2024-04-10T03:03:53.199493+0000","last_active":"2024-04-10T03:04:36.378970+0000","last_peered":"2024-04-10T03:04:36.378970+0000","last_clean":"2024-04-10T03:04:36.378970+0000","last_became_active":"2024-04-10T03:03:53.199197+0000","last_became_peered":"2024-04-10T03:03:53.199197+0000","last_unstale":"2024-04-10T03:04:36.378970+0000","last_undegraded":"2024-04-10T03:04:36.378970+0000","last_fullsized":"2024-04-10T03:04:36.378970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:48:23.646373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379104+0000","last_change":"2024-04-10T03:03:52.176719+0000","last_active":"2024-04-10T03:04:36.379104+0000","last_peered":"2024-04-10T03:04:36.379104+0000","last_clean":"2024-04-10T03:04:36.379104+0000","last_became_active":"2024-04-10T03:03:52.176341+0000","last_became_peered":"2024-04-10T03:03:52.176341+0000","last_unstale":"2024-04-10T03:04:36.379104+0000","last_undegraded":"2024-04-10T03:04:36.379104+0000","last_fullsized":"2024-04-10T03:04:36.379104+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:43.066934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377806+0000","last_change":"2024-04-10T03:03:53.193214+0000","last_active":"2024-04-10T03:04:36.377806+0000","last_peered":"2024-04-10T03:04:36.377806+0000","last_clean":"2024-04-10T03:04:36.377806+0000","last_became_active":"2024-04-10T03:03:53.192937+0000","last_became_peered":"2024-04-10T03:03:53.192937+0000","last_unstale":"2024-04-10T03:04:36.377806+0000","last_undegraded":"2024-04-10T03:04:36.377806+0000","last_fullsized":"2024-04-10T03:04:36.377806+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:26:42.349897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.527079+0000","last_change":"2024-04-10T03:03:52.174299+0000","last_active":"2024-04-10T03:04:36.527079+0000","last_peered":"2024-04-10T03:04:36.527079+0000","last_clean":"2024-04-10T03:04:36.527079+0000","last_became_active":"2024-04-10T03:03:52.174019+0000","last_became_peered":"2024-04-10T03:03:52.174019+0000","last_unstale":"2024-04-10T03:04:36.527079+0000","last_undegraded":"2024-04-10T03:04:36.527079+0000","last_fullsized":"2024-04-10T03:04:36.527079+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:20:26.352214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381121+0000","last_change":"2024-04-10T03:03:53.201699+0000","last_active":"2024-04-10T03:04:36.381121+0000","last_peered":"2024-04-10T03:04:36.381121+0000","last_clean":"2024-04-10T03:04:36.381121+0000","last_became_active":"2024-04-10T03:03:53.201439+0000","last_became_peered":"2024-04-10T03:03:53.201439+0000","last_unstale":"2024-04-10T03:04:36.381121+0000","last_undegraded":"2024-04-10T03:04:36.381121+0000","last_fullsized":"2024-04-10T03:04:36.381121+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:30.816143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.376771+0000","last_change":"2024-04-10T03:03:52.172115+0000","last_active":"2024-04-10T03:04:36.376771+0000","last_peered":"2024-04-10T03:04:36.376771+0000","last_clean":"2024-04-10T03:04:36.376771+0000","last_became_active":"2024-04-10T03:03:52.171799+0000","last_became_peered":"2024-04-10T03:03:52.171799+0000","last_unstale":"2024-04-10T03:04:36.376771+0000","last_undegraded":"2024-04-10T03:04:36.376771+0000","last_fullsized":"2024-04-10T03:04:36.376771+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:23:28.186006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-10T03:04:38.067060+0000","last_change":"2024-04-10T03:04:38.067060+0000","last_active":"2024-04-10T03:04:38.067060+0000","last_peered":"2024-04-10T03:04:38.067060+0000","last_clean":"2024-04-10T03:04:38.067060+0000","last_became_active":"2024-04-10T03:04:38.066712+0000","last_became_peered":"2024-04-10T03:04:38.066712+0000","last_unstale":"2024-04-10T03:04:38.067060+0000","last_undegraded":"2024-04-10T03:04:38.067060+0000","last_fullsized":"2024-04-10T03:04:38.067060+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:40:30.850888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379295+0000","last_change":"2024-04-10T03:03:52.178805+0000","last_active":"2024-04-10T03:04:36.379295+0000","last_peered":"2024-04-10T03:04:36.379295+0000","last_clean":"2024-04-10T03:04:36.379295+0000","last_became_active":"2024-04-10T03:03:52.178486+0000","last_became_peered":"2024-04-10T03:03:52.178486+0000","last_unstale":"2024-04-10T03:04:36.379295+0000","last_undegraded":"2024-04-10T03:04:36.379295+0000","last_fullsized":"2024-04-10T03:04:36.379295+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:05:02.973862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.383732+0000","last_change":"2024-04-10T03:03:53.199352+0000","last_active":"2024-04-10T03:04:36.383732+0000","last_peered":"2024-04-10T03:04:36.383732+0000","last_clean":"2024-04-10T03:04:36.383732+0000","last_became_active":"2024-04-10T03:03:53.199032+0000","last_became_peered":"2024-04-10T03:03:53.199032+0000","last_unstale":"2024-04-10T03:04:36.383732+0000","last_undegraded":"2024-04-10T03:04:36.383732+0000","last_fullsized":"2024-04-10T03:04:36.383732+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:02:50.425776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.382415+0000","last_change":"2024-04-10T03:03:52.180915+0000","last_active":"2024-04-10T03:04:36.382415+0000","last_peered":"2024-04-10T03:04:36.382415+0000","last_clean":"2024-04-10T03:04:36.382415+0000","last_became_active":"2024-04-10T03:03:52.180518+0000","last_became_peered":"2024-04-10T03:03:52.180518+0000","last_unstale":"2024-04-10T03:04:36.382415+0000","last_undegraded":"2024-04-10T03:04:36.382415+0000","last_fullsized":"2024-04-10T03:04:36.382415+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:55:36.302101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.379198+0000","last_change":"2024-04-10T03:03:53.204664+0000","last_active":"2024-04-10T03:04:36.379198+0000","last_peered":"2024-04-10T03:04:36.379198+0000","last_clean":"2024-04-10T03:04:36.379198+0000","last_became_active":"2024-04-10T03:03:53.204321+0000","last_became_peered":"2024-04-10T03:03:53.204321+0000","last_unstale":"2024-04-10T03:04:36.379198+0000","last_undegraded":"2024-04-10T03:04:36.379198+0000","last_fullsized":"2024-04-10T03:04:36.379198+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:13:16.538695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.377721+0000","last_change":"2024-04-10T03:03:52.182973+0000","last_active":"2024-04-10T03:04:36.377721+0000","last_peered":"2024-04-10T03:04:36.377721+0000","last_clean":"2024-04-10T03:04:36.377721+0000","last_became_active":"2024-04-10T03:03:52.182697+0000","last_became_peered":"2024-04-10T03:03:52.182697+0000","last_unstale":"2024-04-10T03:04:36.377721+0000","last_undegraded":"2024-04-10T03:04:36.377721+0000","last_fullsized":"2024-04-10T03:04:36.377721+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:00:38.926901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.381808+0000","last_change":"2024-04-10T03:03:53.206997+0000","last_active":"2024-04-10T03:04:36.381808+0000","last_peered":"2024-04-10T03:04:36.381808+0000","last_clean":"2024-04-10T03:04:36.381808+0000","last_became_active":"2024-04-10T03:03:53.206693+0000","last_became_peered":"2024-04-10T03:03:53.206693+0000","last_unstale":"2024-04-10T03:04:36.381808+0000","last_undegraded":"2024-04-10T03:04:36.381808+0000","last_fullsized":"2024-04-10T03:04:36.381808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:50:16.496041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-10T03:04:36.378809+0000","last_change":"2024-04-10T03:03:52.172387+0000","last_active":"2024-04-10T03:04:36.378809+0000","last_peered":"2024-04-10T03:04:36.378809+0000","last_clean":"2024-04-10T03:04:36.378809+0000","last_became_active":"2024-04-10T03:03:52.172133+0000","last_became_peered":"2024-04-10T03:03:52.172133+0000","last_unstale":"2024-04-10T03:04:36.378809+0000","last_undegraded":"2024-04-10T03:04:36.378809+0000","last_fullsized":"2024-04-10T03:04:36.378809+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:15:29.496387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":905463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66400000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.44700000000000001}]},{"osd":1,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.60299999999999998,"15min":0.56999999999999995},"min":{"1min":0.34200000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.4139999999999997,"5min":6.4139999999999997,"15min":6.4139999999999997},"last":0.499},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.53400000000000003,"15min":0.52800000000000002},"min":{"1min":0.38700000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.629}]},{"osd":2,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.73599999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.58699999999999997,"15min":0.59199999999999997},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.69499999999999995,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.58899999999999997}]},{"osd":3,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54100000000000004,"15min":0.53600000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.57799999999999996,"15min":0.58599999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68500000000000005,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.54500000000000004}]},{"osd":4,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.42499999999999999,"15min":0.41499999999999998},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.32000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.60199999999999998}]},{"osd":5,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.46800000000000003,"15min":0.45400000000000001},"min":{"1min":0.36099999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.621},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.50700000000000001,"15min":0.495},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.64700000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.34899999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.54200000000000004,"15min":0.53400000000000003},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.67800000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":905052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69199999999999995,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.52900000000000003}]},{"osd":1,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.52300000000000002,"15min":0.51300000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77700000000000002,"5min":1.012,"15min":1.012},"last":0.65500000000000003}]},{"osd":2,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76100000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.52900000000000003},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40100000000000002}]},{"osd":3,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54100000000000004,"15min":0.53200000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.40000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47199999999999998}]},{"osd":4,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45000000000000001,"15min":0.433},"min":{"1min":0.29399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.66000000000000003,"5min":0.752,"15min":0.752},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.46000000000000002,"15min":0.432},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.72099999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.60599999999999998}]},{"osd":5,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.45300000000000001,"15min":0.42299999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.75900000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.495,"15min":0.46899999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.495}]},{"osd":7,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.52200000000000002,"15min":0.51100000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63300000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":313650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.315},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.667,"5min":1.667,"15min":1.667},"last":0.372},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.376,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.8160000000000001,"5min":1.8160000000000001,"15min":1.8160000000000001},"last":0.438}]},{"osd":2,"last update":"Wed Apr 10 03:04:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.498,"5min":1.452,"15min":1.452},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.377,"15min":0.374},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52300000000000002,"5min":1.365,"15min":1.365},"last":0.42199999999999999}]},{"osd":3,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.44800000000000001,"15min":0.45400000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.53400000000000003,"5min":1.395,"15min":1.395},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.44,"15min":0.433},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.55000000000000004,"5min":1.423,"15min":1.423},"last":0.46100000000000002}]},{"osd":4,"last update":"Wed Apr 10 03:03:58 2024","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.71299999999999997,"15min":0.60899999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":16.536999999999999,"5min":16.536999999999999,"15min":16.536999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73699999999999999,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.60999999999999999}]},{"osd":5,"last update":"Wed Apr 10 03:04:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76700000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78100000000000003,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.55700000000000005}]},{"osd":6,"last update":"Wed Apr 10 03:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.63400000000000001,"15min":0.60499999999999998},"min":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.61199999999999999,"15min":0.61099999999999999},"min":{"1min":0.50800000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.57499999999999996}]},{"osd":7,"last update":"Wed Apr 10 03:04:07 2024","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.76400000000000001,"15min":0.64700000000000002},"min":{"1min":0.44400000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":16.507999999999999,"5min":16.507999999999999,"15min":16.507999999999999},"last":0.745},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70599999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.69499999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":904610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Apr 10 03:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.36299999999999999,"15min":0.32000000000000001},"min":{"1min":0.23400000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.379,"15min":0.34100000000000003},"min":{"1min":0.22900000000000001,"5min":0.216,"15min":0.216},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.26100000000000001}]},{"osd":2,"last update":"Wed Apr 10 03:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.312},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40500000000000003,"15min":0.40200000000000002},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64700000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.34200000000000003}]},{"osd":3,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.38},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.42199999999999999,"15min":0.432},"min":{"1min":0.26600000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70299999999999996,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.28299999999999997}]},{"osd":4,"last update":"Wed Apr 10 03:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51900000000000002}]},{"osd":5,"last update":"Wed Apr 10 03:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.57199999999999995,"15min":0.56799999999999995},"min":{"1min":0.39700000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.91000000000000003,"5min":1.155,"15min":1.155},"last":0.47999999999999998}]},{"osd":6,"last update":"Wed Apr 10 03:04:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.74299999999999999,"15min":0.70099999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0369999999999999,"5min":9.6449999999999996,"15min":9.6449999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.70899999999999996,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.022,"5min":9.6319999999999997,"15min":9.6319999999999997},"last":0.46100000000000002}]},{"osd":7,"last update":"Wed Apr 10 03:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.626,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.97499999999999998,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.502},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.42399999999999999,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.84399999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.495}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35699999999999998,"15min":0.33700000000000002},"min":{"1min":0.22700000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.374,"15min":0.35399999999999998},"min":{"1min":0.28199999999999997,"5min":0.222,"15min":0.222},"max":{"1min":0.51500000000000001,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.28100000000000003}]},{"osd":1,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38600000000000001,"15min":0.36399999999999999},"min":{"1min":0.26500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.39900000000000002},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.36099999999999999}]},{"osd":3,"last update":"Wed Apr 10 03:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.45000000000000001,"15min":0.46200000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.42999999999999999,"15min":0.44},"min":{"1min":0.307,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.31900000000000001}]},{"osd":4,"last update":"Wed Apr 10 03:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56599999999999995,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.64600000000000002,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.57899999999999996},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.443}]},{"osd":5,"last update":"Wed Apr 10 03:04:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.55900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77400000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.46400000000000002}]},{"osd":6,"last update":"Wed Apr 10 03:04:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61499999999999999,"15min":0.61299999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.433},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60399999999999998,"15min":0.60499999999999998},"min":{"1min":0.41699999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.748,"5min":0.997,"15min":0.997},"last":0.51400000000000001}]},{"osd":7,"last update":"Wed Apr 10 03:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57599999999999996,"15min":0.57299999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.45300000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":313740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.373,"15min":0.39700000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.39700000000000002,"15min":0.42299999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.51000000000000001,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.36399999999999999}]},{"osd":1,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.39100000000000001,"15min":0.42999999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.46999999999999997,"5min":0.752,"15min":0.752},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.43099999999999999,"15min":0.46100000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.52400000000000002,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.42499999999999999}]},{"osd":2,"last update":"Wed Apr 10 03:04:38 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.41199999999999998,"15min":0.44800000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.52100000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.45500000000000002,"15min":0.49199999999999999},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.50700000000000001,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.45900000000000002}]},{"osd":4,"last update":"Wed Apr 10 03:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71099999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.754},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57799999999999996,"15min":0.58799999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69999999999999996,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.78800000000000003}]},{"osd":5,"last update":"Wed Apr 10 03:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.39600000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.75,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.61799999999999999}]},{"osd":6,"last update":"Wed Apr 10 03:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81100000000000005,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.86899999999999999,"15min":0.998},"min":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.755,"5min":9.718,"15min":9.718},"last":0.68700000000000006}]},{"osd":7,"last update":"Wed Apr 10 03:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77300000000000002,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.012,"15min":1.0800000000000001},"min":{"1min":0.41799999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.84199999999999997,"5min":9.9160000000000004,"15min":9.9160000000000004},"last":0.71099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":737280,"data_stored":315083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51200000000000001,"15min":0.501},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.70699999999999996,"5min":0.745,"15min":0.745},"last":0.63800000000000001}]},{"osd":1,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.39400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.33100000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77000000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55200000000000005}]},{"osd":2,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50800000000000001,"15min":0.5},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.38800000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.65100000000000002}]},{"osd":3,"last update":"Wed Apr 10 03:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.45000000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.69699999999999995,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Wed Apr 10 03:04:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.39900000000000002,"15min":0.38100000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.436,"15min":0.40999999999999998},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53300000000000003}]},{"osd":6,"last update":"Wed Apr 10 03:04:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.52300000000000002,"15min":0.497},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.46400000000000002,"15min":0.45100000000000001},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.437}]},{"osd":7,"last update":"Wed Apr 10 03:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.50800000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757293,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":314290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":0.76800000000000002,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.65200000000000002}]},{"osd":1,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71399999999999997,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.626},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.58499999999999996,"15min":0.55800000000000005},"min":{"1min":0.41999999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":0.61599999999999999}]},{"osd":2,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.5270000000000001,"5min":4.5270000000000001,"15min":4.5270000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55100000000000005,"15min":0.55800000000000005},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73099999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.60299999999999998}]},{"osd":3,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.66700000000000004,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.55100000000000005,"15min":0.54200000000000004},"min":{"1min":0.41599999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73999999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.70399999999999996}]},{"osd":4,"last update":"Wed Apr 10 03:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.437,"15min":0.42499999999999999},"min":{"1min":0.34200000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45900000000000002,"15min":0.44800000000000001},"min":{"1min":0.32400000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.76700000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.53900000000000003}]},{"osd":6,"last update":"Wed Apr 10 03:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.46100000000000002,"15min":0.44500000000000001},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.48799999999999999,"15min":0.46800000000000003},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.44}]},{"osd":7,"last update":"Wed Apr 10 03:04:12 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52200000000000002,"15min":0.51900000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70599999999999996,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78500000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:04:53.439 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-10T03:04:53.439 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-10T03:04:53.439 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-10T03:04:53.439 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:04:53.694 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:53 smithi152 ceph-mon[33523]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:53.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:53 smithi102 ceph-mon[39651]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:53.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:53 smithi102 ceph-mon[29301]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:54.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:54 smithi152 ceph-mon[33523]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:54.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:54 smithi102 ceph-mon[29301]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:54 smithi102 ceph-mon[39651]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:04:55.535 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:04:55.536 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-10T03:04:55.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:55 smithi152 ceph-mon[33523]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:55.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:55 smithi102 ceph-mon[29301]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:55.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:55 smithi102 ceph-mon[39651]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:56.175 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-10T03:04:56.176 INFO:teuthology.run_tasks:Running task print... 2024-04-10T03:04:56.187 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-10T03:04:56.187 INFO:teuthology.run_tasks:Running task parallel... 2024-04-10T03:04:56.196 INFO:teuthology.task.parallel:starting parallel... 2024-04-10T03:04:56.197 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-10T03:04:56.197 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-10T03:04:56.198 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-10T03:04:56.199 INFO:tasks.radosbench:Beginning radosbench... 2024-04-10T03:04:56.199 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-04-10T03:04:56.199 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool create unique_pool_0 16 2024-04-10T03:04:56.200 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-10T03:04:56.200 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-10T03:04:56.200 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T03:04:56.200 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-10T03:04:56.480 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:56.481 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:04:56.732 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:56 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2346955331' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:04:56.732 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:56 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2346955331' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:04:56.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:56 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2346955331' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:04:57.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:57 smithi152 ceph-mon[33523]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:57.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:57 smithi102 ceph-mon[29301]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:57.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:57 smithi102 ceph-mon[39651]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:58] ENGINE Bus STOPPING 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:58] ENGINE Bus STOPPED 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:58] ENGINE Bus STARTING 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:59] ENGINE Serving on http://:::9283 2024-04-10T03:04:59.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:04:59] ENGINE Bus STARTED 2024-04-10T03:04:59.429 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:04:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:04:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:04:59.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:59 smithi152 ceph-mon[33523]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:59 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' 2024-04-10T03:04:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:04:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[39651]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[29301]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[29301]: from='client.? ' entity='client.admin' 2024-04-10T03:04:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:04:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:00.129 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-10T03:05:00.623 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:00.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:00.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:00.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:00.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:00.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:00.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:00 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:00 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:01.646 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_0' created 2024-04-10T03:05:01.647 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:01 smithi102 ceph-mon[29301]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:01.647 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:01 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-10T03:05:01.647 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:01 smithi102 ceph-mon[39651]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:01.647 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:01 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-10T03:05:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:01 smithi152 ceph-mon[33523]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:01 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-10T03:05:02.552 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-04-10T03:05:02.681 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:02] "GET /metrics HTTP/1.1" 200 239133 "" "Prometheus/2.43.0" 2024-04-10T03:05:02.681 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-10T03:05:02.681 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[29301]: osdmap e71: 8 total, 8 up, 8 in 2024-04-10T03:05:02.681 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[29301]: pgmap v88: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:02.682 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-10T03:05:02.682 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[39651]: osdmap e71: 8 total, 8 up, 8 in 2024-04-10T03:05:02.682 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:02 smithi102 ceph-mon[39651]: pgmap v88: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:02 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3694128119' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-10T03:05:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:02 smithi152 ceph-mon[33523]: osdmap e71: 8 total, 8 up, 8 in 2024-04-10T03:05:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:02 smithi152 ceph-mon[33523]: pgmap v88: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:03.050 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:03.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:03 smithi152 ceph-mon[33523]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:05:03.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:03 smithi152 ceph-mon[33523]: osdmap e72: 8 total, 8 up, 8 in 2024-04-10T03:05:03.967 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:03 smithi102 ceph-mon[29301]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:05:03.967 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:03 smithi102 ceph-mon[29301]: osdmap e72: 8 total, 8 up, 8 in 2024-04-10T03:05:03.967 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:03 smithi102 ceph-mon[39651]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:05:03.967 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:03 smithi102 ceph-mon[39651]: osdmap e72: 8 total, 8 up, 8 in 2024-04-10T03:05:04.436 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-10T03:05:04.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:04 smithi102 ceph-mon[39651]: osdmap e73: 8 total, 8 up, 8 in 2024-04-10T03:05:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:04 smithi102 ceph-mon[39651]: pgmap v91: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:04 smithi102 ceph-mon[29301]: osdmap e73: 8 total, 8 up, 8 in 2024-04-10T03:05:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:04 smithi102 ceph-mon[29301]: pgmap v91: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:04 smithi152 ceph-mon[33523]: osdmap e73: 8 total, 8 up, 8 in 2024-04-10T03:05:05.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:04 smithi152 ceph-mon[33523]: pgmap v91: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:05.413 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:06.340 INFO:teuthology.orchestra.run.smithi102.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:06.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:06 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:05:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:06.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:07.648 INFO:teuthology.orchestra.run.smithi102.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:07.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[29301]: pgmap v92: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:07.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:05:07.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[29301]: osdmap e74: 8 total, 8 up, 8 in 2024-04-10T03:05:07.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[39651]: pgmap v92: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:07.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:05:07.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:07 smithi102 ceph-mon[39651]: osdmap e74: 8 total, 8 up, 8 in 2024-04-10T03:05:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:07 smithi152 ceph-mon[33523]: pgmap v92: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:07 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3729774894' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:05:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:07 smithi152 ceph-mon[33523]: osdmap e74: 8 total, 8 up, 8 in 2024-04-10T03:05:08.612 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:05:08.901 DEBUG:teuthology.orchestra.run.smithi102:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-04-10T03:05:08.903 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-10T03:05:08.903 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-10T03:05:08.913 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:hints = 1 2024-04-10T03:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-10T03:05:08.957 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object prefix: benchmark_data_smithi102_79578 2024-04-10T03:05:08.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:05:08.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 0 0 0 0 0 0 - 0 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:08 smithi152 ceph-mon[33523]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:09.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:09.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:09.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[29301]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:09.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a866975a60f6dc859246f6f19fd8ddb4db2cc2e 2024-04-10T03:05:09.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:08 smithi102 ceph-mon[39651]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:09.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:09.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 1 15 3846 3831 239.423 239.438 0.00349203 0.00416225 2024-04-10T03:05:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:09 smithi152 ceph-mon[33523]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:05:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:09 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:05:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:09 smithi102 ceph-mon[29301]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:05:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:09 smithi102 ceph-mon[29301]: Cluster is now healthy 2024-04-10T03:05:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:09 smithi102 ceph-mon[39651]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:05:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:09 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:05:10.853 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:10 smithi102 ceph-mon[39651]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:10.853 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:10 smithi102 ceph-mon[29301]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:10.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 2 16 7292 7276 227.357 215.312 0.00231486 0.00438992 2024-04-10T03:05:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:10 smithi152 ceph-mon[33523]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:05:11.766 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:05:11.766 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (89s) 44s ago 2m 15.2M - 0.25.0 c8568f914cd2 4f5410539eed 2024-04-10T03:05:11.766 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 running (2m) 60s ago 2m 84.4M - 9.4.7 954c08fa6188 64967280ffee 2024-04-10T03:05:11.766 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (73s) 44s ago 72s 12.4M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:05:11.766 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (70s) 60s ago 70s 13.4M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443 running (6m) 60s ago 6m 432M - 17.2.7-827-gd3b5060a 21de9b7369ad a1843f19df0f 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:9283 running (8m) 44s ago 8m 475M - 17.2.7-827-gd3b5060a 21de9b7369ad 8be8627dd288 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (8m) 44s ago 8m 43.0M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 2b567b259d3d 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 running (6m) 60s ago 6m 38.2M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 7ecfd799af6d 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 running (6m) 44s ago 6m 34.4M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 8f8b8b481e8d 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (116s) 44s ago 113s 13.2M - 1.5.0 0da6a335fe13 02931ad901dc 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (110s) 60s ago 110s 10.2M - 1.5.0 0da6a335fe13 6f30ab34e787 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (6m) 44s ago 6m 41.9M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 984949346b35 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (5m) 44s ago 5m 42.5M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 13648c191186 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (5m) 44s ago 5m 43.1M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad f10dfba3072f 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (4m) 44s ago 4m 40.7M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 6f7249bbf2b8 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (4m) 60s ago 4m 42.1M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (4m) 60s ago 4m 40.1M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (3m) 60s ago 3m 39.8M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:05:11.767 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (3m) 60s ago 3m 40.1M 4016M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:05:11.768 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (84s) 60s ago 2m 35.2M - 2.43.0 a07b618ecd1d a2d3e5fc2d74 2024-04-10T03:05:11.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 3 16 10347 10331 215.212 190.938 0.00468119 0.00463995 2024-04-10T03:05:12.748 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:12] "GET /metrics HTTP/1.1" 200 241480 "" "Prometheus/2.43.0" 2024-04-10T03:05:12.793 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-10T03:05:12.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 4 16 13482 13466 210.384 195.938 0.00872896 0.00474481 2024-04-10T03:05:13.322 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:13.468 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:13 smithi102 ceph-mon[29301]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:13.468 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:13 smithi102 ceph-mon[29301]: pgmap v96: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-10T03:05:13.468 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:13 smithi102 ceph-mon[39651]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:13.468 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:13 smithi102 ceph-mon[39651]: pgmap v96: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-10T03:05:13.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:13 smithi152 ceph-mon[33523]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:05:13.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:13 smithi152 ceph-mon[33523]: pgmap v96: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-10T03:05:13.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 5 16 16326 16310 203.854 177.75 0.00466954 0.00489965 2024-04-10T03:05:14.959 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 6 16 19240 19224 200.229 182.125 0.00736472 0.00498873 2024-04-10T03:05:15.307 INFO:teuthology.orchestra.run.smithi102.stdout:wait for minority of mons to upgrade 2024-04-10T03:05:15.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:15 smithi152 ceph-mon[33523]: pgmap v97: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:15 smithi102 ceph-mon[29301]: pgmap v97: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:15.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:15 smithi102 ceph-mon[39651]: pgmap v97: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:15.958 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 7 16 21708 21692 193.657 154.25 0.00111054 0.00513494 2024-04-10T03:05:16.959 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 8 16 23924 23908 186.761 138.5 0.0120052 0.00534886 2024-04-10T03:05:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:17 smithi152 ceph-mon[33523]: pgmap v98: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:17 smithi102 ceph-mon[39651]: pgmap v98: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:17.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:17 smithi102 ceph-mon[29301]: pgmap v98: 89 pgs: 89 active+clean; 673 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-10T03:05:17.959 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 9 16 26084 26068 181.008 135 0.00271136 0.00551801 2024-04-10T03:05:18.959 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 10 16 28056 28040 175.231 123.25 0.00481944 0.00570146 2024-04-10T03:05:19.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:19 smithi152 ceph-mon[33523]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-10T03:05:19.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:19.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:19 smithi102 ceph-mon[29301]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-10T03:05:19.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:19 smithi102 ceph-mon[39651]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-10T03:05:19.959 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 11 16 29736 29720 168.835 105 0.00569577 0.00591784 2024-04-10T03:05:20.960 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 12 15 31966 31951 166.384 139.438 0.00575407 0.00600479 2024-04-10T03:05:21.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:21 smithi152 ceph-mon[33523]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-10T03:05:21.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:21 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:21.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:21 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:21.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[29301]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-10T03:05:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[39651]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-04-10T03:05:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:21 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:21.718 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-10T03:05:21.960 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 13 16 33859 33843 162.681 118.25 0.00453183 0.00614084 2024-04-10T03:05:22.203 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:05:22.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:22] "GET /metrics HTTP/1.1" 200 242429 "" "Prometheus/2.43.0" 2024-04-10T03:05:22.960 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 14 16 35803 35787 159.739 121.5 0.00954894 0.00625505 2024-04-10T03:05:23.227 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:23 smithi102 ceph-mon[29301]: pgmap v101: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.84k op/s 2024-04-10T03:05:23.227 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:23 smithi102 ceph-mon[39651]: pgmap v101: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.84k op/s 2024-04-10T03:05:23.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:23 smithi152 ceph-mon[33523]: pgmap v101: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.84k op/s 2024-04-10T03:05:23.960 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 15 16 37942 37926 158.001 133.688 0.00299461 0.0063239 2024-04-10T03:05:24.961 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 16 16 39918 39902 155.835 123.5 0.0147384 0.00641133 2024-04-10T03:05:25.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:25 smithi152 ceph-mon[33523]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:25.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:25 smithi102 ceph-mon[29301]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:25 smithi102 ceph-mon[39651]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:25.961 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 17 16 41841 41825 153.734 120.188 0.00442014 0.0065005 2024-04-10T03:05:26.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:26 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1690297847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:26.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:26 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1690297847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:26.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:26 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1690297847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:26.962 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 18 16 43853 43837 152.179 125.75 0.0122926 0.00656628 2024-04-10T03:05:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:27 smithi152 ceph-mon[33523]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:27.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:27 smithi102 ceph-mon[29301]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:27.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:27 smithi102 ceph-mon[39651]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-04-10T03:05:27.963 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 19 16 45738 45722 150.36 117.812 0.0106125 0.00664638 2024-04-10T03:05:28.964 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:05:28.967207+0000 min lat: 0.00111054 max lat: 0.154473 avg lat: 0.00672796 2024-04-10T03:05:28.964 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:05:28.964 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 20 16 47563 47547 148.538 114.062 0.0068736 0.00672796 2024-04-10T03:05:29.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:29 smithi152 ceph-mon[33523]: pgmap v104: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s 2024-04-10T03:05:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:29 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1059305091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:29.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:29.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:29 smithi102 ceph-mon[29301]: pgmap v104: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s 2024-04-10T03:05:29.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:29 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1059305091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:29.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:29 smithi102 ceph-mon[39651]: pgmap v104: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s 2024-04-10T03:05:29.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:29 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1059305091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:29.965 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 21 16 49415 49399 146.966 115.75 0.00713315 0.00679948 2024-04-10T03:05:30.966 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 22 16 51109 51093 145.093 105.875 0.0132083 0.0068869 2024-04-10T03:05:31.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:31 smithi152 ceph-mon[33523]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-10T03:05:31.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:31 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1183436793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:31.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:31 smithi102 ceph-mon[29301]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-10T03:05:31.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:31 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1183436793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:31.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:31 smithi102 ceph-mon[39651]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-10T03:05:31.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:31 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1183436793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:31.966 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 23 16 52879 52863 143.594 110.625 0.0101051 0.00695967 2024-04-10T03:05:32.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:32] "GET /metrics HTTP/1.1" 200 242429 "" "Prometheus/2.43.0" 2024-04-10T03:05:32.967 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 24 16 54630 54614 142.171 109.438 0.0250511 0.00702805 2024-04-10T03:05:33.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:33 smithi152 ceph-mon[33523]: pgmap v106: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2024-04-10T03:05:33.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:33 smithi102 ceph-mon[29301]: pgmap v106: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2024-04-10T03:05:33.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:33 smithi102 ceph-mon[39651]: pgmap v106: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 150 MiB/s wr, 2.41k op/s 2024-04-10T03:05:33.969 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 25 16 56113 56097 140.179 92.6875 0.007034 0.00712842 2024-04-10T03:05:34.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:34 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3791316298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:34.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:34 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3791316298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:34.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:34 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3791316298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:34.969 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 26 16 57558 57542 138.261 90.3125 0.0152964 0.00722686 2024-04-10T03:05:35.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:35 smithi102 ceph-mon[29301]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:35.258 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:35 smithi102 ceph-mon[39651]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:35.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:35 smithi152 ceph-mon[33523]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:35.970 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 27 16 58885 58869 136.21 82.9375 0.0237291 0.00733469 2024-04-10T03:05:36.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:36.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:36.544 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:36.970 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 28 16 60174 60158 134.223 80.5625 0.0093714 0.00744536 2024-04-10T03:05:37.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:37 smithi152 ceph-mon[33523]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:37.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:37 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3835465887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:37.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:37 smithi102 ceph-mon[39651]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:37.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:37 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3835465887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:37.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:37 smithi102 ceph-mon[29301]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-10T03:05:37.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:37 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3835465887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:37.970 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 29 16 61736 61720 132.96 97.625 0.00466052 0.00751608 2024-04-10T03:05:38.970 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 30 16 63113 63097 131.397 86.0625 0.00931557 0.00760618 2024-04-10T03:05:39.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:39 smithi152 ceph-mon[33523]: pgmap v109: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:05:39.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:39 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/983232145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:39.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:39.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:39 smithi102 ceph-mon[39651]: pgmap v109: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:05:39.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:39 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/983232145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:39 smithi102 ceph-mon[29301]: pgmap v109: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:05:39.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:39 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/983232145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:39.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 31 16 64548 64532 130.049 89.6875 0.00400723 0.00768427 2024-04-10T03:05:40.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 32 16 66033 66017 128.886 92.8125 0.00987984 0.00775441 2024-04-10T03:05:41.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:41 smithi152 ceph-mon[33523]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:05:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:41 smithi102 ceph-mon[39651]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:05:41.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:41 smithi102 ceph-mon[29301]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:05:41.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 33 16 67401 67385 127.571 85.5 0.00890101 0.00783455 2024-04-10T03:05:42.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:42 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3762598278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:42.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:42] "GET /metrics HTTP/1.1" 200 242610 "" "Prometheus/2.43.0" 2024-04-10T03:05:42.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:42 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3762598278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:42 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3762598278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:42.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 34 16 68925 68909 126.621 95.25 0.00639157 0.00789277 2024-04-10T03:05:43.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:43 smithi102 ceph-mon[29301]: pgmap v111: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-04-10T03:05:43.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:43 smithi102 ceph-mon[39651]: pgmap v111: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-04-10T03:05:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:43 smithi152 ceph-mon[33523]: pgmap v111: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-04-10T03:05:43.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 35 16 70292 70276 125.444 85.4375 0.00737965 0.00796647 2024-04-10T03:05:44.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 36 16 71486 71470 124.033 74.625 0.0113252 0.00805771 2024-04-10T03:05:45.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:45 smithi102 ceph-mon[39651]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:45 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3609563551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:45.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:45 smithi102 ceph-mon[29301]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:45.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:45 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3609563551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:45.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:45 smithi152 ceph-mon[33523]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:45.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:45 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3609563551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:45.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 37 16 72720 72704 122.765 77.125 0.0116963 0.00814104 2024-04-10T03:05:46.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 38 16 73805 73789 121.319 67.8125 0.0153712 0.00823811 2024-04-10T03:05:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:47 smithi102 ceph-mon[29301]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:47 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1736827807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:47.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:47 smithi102 ceph-mon[39651]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:47.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:47 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1736827807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:47 smithi152 ceph-mon[33523]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-04-10T03:05:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:47 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1736827807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:47.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 39 16 75152 75136 120.367 84.1875 0.0091915 0.00830302 2024-04-10T03:05:48.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:05:48.974959+0000 min lat: 0.00111054 max lat: 0.157392 avg lat: 0.00834875 2024-04-10T03:05:48.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:05:48.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 40 16 76652 76636 119.702 93.75 0.00938651 0.00834875 2024-04-10T03:05:49.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:49 smithi152 ceph-mon[33523]: pgmap v114: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-10T03:05:49.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:49.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:49 smithi102 ceph-mon[39651]: pgmap v114: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-10T03:05:49.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:49 smithi102 ceph-mon[29301]: pgmap v114: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-04-10T03:05:49.971 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 41 16 78038 78022 118.895 86.625 0.00643063 0.00840589 2024-04-10T03:05:50.555 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:50 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4256724004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:50.555 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:50 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4256724004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:50 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4256724004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:50.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 42 16 79479 79463 118.209 90.0625 0.0118933 0.00845485 2024-04-10T03:05:51.620 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:51 smithi102 ceph-mon[29301]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-10T03:05:51.620 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:51.620 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:51 smithi102 ceph-mon[39651]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-10T03:05:51.620 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:51 smithi152 ceph-mon[33523]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-10T03:05:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:05:51.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 43 16 80884 80868 117.502 87.8125 0.00877577 0.00850567 2024-04-10T03:05:52.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:52] "GET /metrics HTTP/1.1" 200 242642 "" "Prometheus/2.43.0" 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: Upgrade: Target is version 19.0.0-2051-g3a866975 (unknown) 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f'] 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: pgmap v116: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f on smithi152 2024-04-10T03:05:52.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4169213890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: Upgrade: Target is version 19.0.0-2051-g3a866975 (unknown) 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f'] 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: pgmap v116: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f on smithi152 2024-04-10T03:05:52.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:52 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4169213890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:52.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 44 16 82146 82130 116.624 78.875 0.00858818 0.00857039 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: Upgrade: Target is version 19.0.0-2051-g3a866975 (unknown) 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f'] 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: pgmap v116: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f on smithi152 2024-04-10T03:05:53.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:52 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4169213890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:53.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 45 16 83341 83325 115.692 74.6875 0.00907383 0.00863865 2024-04-10T03:05:54.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 46 16 84504 84488 114.757 72.6875 0.0206702 0.00870918 2024-04-10T03:05:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:55 smithi152 ceph-mon[33523]: pgmap v117: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-04-10T03:05:55.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:55 smithi102 ceph-mon[29301]: pgmap v117: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-04-10T03:05:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:55 smithi102 ceph-mon[39651]: pgmap v117: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-04-10T03:05:55.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 47 16 85685 85669 113.886 73.8125 0.0281688 0.00877601 2024-04-10T03:05:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:56 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2482844380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:56.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:56 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2482844380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:56 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2482844380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:56.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 48 16 86824 86808 112.997 71.1875 0.0259265 0.00884463 2024-04-10T03:05:57.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:57 smithi152 ceph-mon[33523]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:05:57.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:57 smithi102 ceph-mon[39651]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:05:57.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:57 smithi102 ceph-mon[29301]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:05:57.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 49 16 87966 87950 112.147 71.375 0.0141712 0.00891138 2024-04-10T03:05:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:58 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3586426843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:58 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3586426843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:58.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:58 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3586426843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:05:58.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 50 16 89089 89073 111.308 70.1875 0.00896766 0.00897878 2024-04-10T03:05:59.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:05:59 smithi152 ceph-mon[33523]: pgmap v119: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-10T03:05:59.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:05:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:05:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:05:59.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:05:59 smithi102 ceph-mon[29301]: pgmap v119: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-10T03:05:59.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:05:59 smithi102 ceph-mon[39651]: pgmap v119: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-10T03:05:59.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 51 16 90189 90173 110.474 68.75 0.00772282 0.00904698 2024-04-10T03:06:00.972 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 52 16 91150 91134 109.505 60.0625 0.0206072 0.0091269 2024-04-10T03:06:01.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:01 smithi152 ceph-mon[33523]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-10T03:06:01.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:01 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1407908660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:01.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:01 smithi102 ceph-mon[29301]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-10T03:06:01.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:01 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1407908660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:01.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:01 smithi102 ceph-mon[39651]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-10T03:06:01.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:01 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1407908660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:01.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 53 16 92054 92038 108.505 56.5 0.0256424 0.0092105 2024-04-10T03:06:02.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:02] "GET /metrics HTTP/1.1" 200 242642 "" "Prometheus/2.43.0" 2024-04-10T03:06:02.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 54 16 92869 92853 107.439 50.9375 0.0285258 0.00930234 2024-04-10T03:06:03.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:03 smithi152 ceph-mon[33523]: pgmap v121: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-10T03:06:03.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:03 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2122592656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:03.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:03 smithi102 ceph-mon[29301]: pgmap v121: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-10T03:06:03.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:03 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2122592656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:03.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:03 smithi102 ceph-mon[39651]: pgmap v121: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-10T03:06:03.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:03 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2122592656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:03.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 55 16 93870 93854 106.623 62.5625 0.0102369 0.00937368 2024-04-10T03:06:04.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 56 16 94864 94848 105.828 62.125 0.0126872 0.00944417 2024-04-10T03:06:05.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:05 smithi152 ceph-mon[33523]: pgmap v122: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-10T03:06:05.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:05 smithi102 ceph-mon[39651]: pgmap v122: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-10T03:06:05.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:05 smithi102 ceph-mon[29301]: pgmap v122: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-10T03:06:05.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 57 16 95906 95890 105.114 65.125 0.0103061 0.00950877 2024-04-10T03:06:06.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:06 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/395241443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:06.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:06 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/395241443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:06 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:06.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:06 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/395241443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:06.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:06 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:06.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 58 16 96951 96935 104.428 65.3125 0.00940223 0.00957093 2024-04-10T03:06:07.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:07 smithi152 ceph-mon[33523]: pgmap v123: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-10T03:06:07.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:07 smithi102 ceph-mon[39651]: pgmap v123: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-10T03:06:07.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:07 smithi102 ceph-mon[29301]: pgmap v123: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-04-10T03:06:07.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 59 16 98000 97984 103.769 65.5625 0.0192373 0.00963187 2024-04-10T03:06:08.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:06:08.976658+0000 min lat: 0.00111054 max lat: 0.157392 avg lat: 0.00969347 2024-04-10T03:06:08.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:06:08.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 60 16 99016 99000 103.098 63.5 0.00477876 0.00969347 2024-04-10T03:06:09.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:09 smithi152 ceph-mon[33523]: pgmap v124: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:06:09.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:09 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/232728343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:09.515 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:06:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:09 smithi102 ceph-mon[29301]: pgmap v124: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:06:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:09 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/232728343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:09.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:09 smithi102 ceph-mon[39651]: pgmap v124: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:06:09.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:09 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/232728343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:09.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 61 16 100060 100044 102.478 65.25 0.0143362 0.00975371 2024-04-10T03:06:10.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 62 16 101080 101064 101.853 63.75 0.00762132 0.0098135 2024-04-10T03:06:11.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:11 smithi152 ceph-mon[33523]: pgmap v125: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-04-10T03:06:11.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:11 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3436042850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:11.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:11 smithi102 ceph-mon[39651]: pgmap v125: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-04-10T03:06:11.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:11 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3436042850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:11.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:11 smithi102 ceph-mon[29301]: pgmap v125: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-04-10T03:06:11.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:11 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3436042850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:11.973 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 63 16 102036 102020 101.185 59.75 0.0116608 0.00987848 2024-04-10T03:06:12.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:12] "GET /metrics HTTP/1.1" 200 242736 "" "Prometheus/2.43.0" 2024-04-10T03:06:12.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 64 16 103008 102992 100.553 60.75 0.0133691 0.00993988 2024-04-10T03:06:13.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:13 smithi152 ceph-mon[33523]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-10T03:06:13.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:13 smithi102 ceph-mon[29301]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-10T03:06:13.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:13 smithi102 ceph-mon[39651]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-10T03:06:13.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 65 16 104008 103992 99.9675 62.5 0.0155662 0.00999886 2024-04-10T03:06:14.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:14 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/27992645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:14.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:14 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/27992645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:14.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:14 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/27992645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:14.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 66 16 105039 105023 99.4292 64.4375 0.0292448 0.0100528 2024-04-10T03:06:15.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:15 smithi102 ceph-mon[39651]: pgmap v127: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-10T03:06:15.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:15 smithi102 ceph-mon[29301]: pgmap v127: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-10T03:06:15.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:15 smithi152 ceph-mon[33523]: pgmap v127: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-10T03:06:15.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 67 16 106013 105997 98.8538 60.875 0.00767343 0.0101108 2024-04-10T03:06:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:16 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3531493396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:16 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3531493396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:16 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3531493396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:16.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 68 16 106961 106945 98.2714 59.25 0.0058576 0.0101706 2024-04-10T03:06:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:17 smithi102 ceph-mon[39651]: pgmap v128: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-04-10T03:06:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:17 smithi102 ceph-mon[29301]: pgmap v128: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-04-10T03:06:17.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:17 smithi152 ceph-mon[33523]: pgmap v128: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-04-10T03:06:17.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 69 16 107900 107884 97.6977 58.6875 0.00480591 0.01023 2024-04-10T03:06:18.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 70 16 108800 108784 97.1056 56.25 0.0231895 0.0102932 2024-04-10T03:06:19.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:19 smithi152 ceph-mon[33523]: pgmap v129: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-10T03:06:19.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:19 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2153831551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:19.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:06:19.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:19 smithi102 ceph-mon[39651]: pgmap v129: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-10T03:06:19.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:19 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2153831551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:19.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:19 smithi102 ceph-mon[29301]: pgmap v129: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-10T03:06:19.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:19 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2153831551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:19.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 71 16 109728 109712 96.5548 58 0.0146377 0.0103514 2024-04-10T03:06:20.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 72 16 110643 110627 96.0081 57.1875 0.019662 0.0104106 2024-04-10T03:06:21.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:21 smithi102 ceph-mon[39651]: pgmap v130: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2024-04-10T03:06:21.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:21 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:21 smithi102 ceph-mon[29301]: pgmap v130: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2024-04-10T03:06:21.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:21 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:21 smithi152 ceph-mon[33523]: pgmap v130: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2024-04-10T03:06:21.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:21 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:21.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 73 16 111539 111523 95.46 56 0.0237832 0.0104705 2024-04-10T03:06:22.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2525501595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:22.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2525501595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:22.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:22] "GET /metrics HTTP/1.1" 200 242759 "" "Prometheus/2.43.0" 2024-04-10T03:06:22.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2525501595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:22.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 74 16 112374 112358 94.8753 52.1875 0.0223864 0.0105347 2024-04-10T03:06:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:23 smithi152 ceph-mon[33523]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-04-10T03:06:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:23 smithi102 ceph-mon[39651]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-04-10T03:06:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:23 smithi102 ceph-mon[29301]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-04-10T03:06:23.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 75 16 113307 113291 94.3878 58.3125 0.0220269 0.0105896 2024-04-10T03:06:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:24 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1860396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:24 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1860396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:24 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1860396850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:24.974 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 76 16 114205 114189 93.8844 56.125 0.0345528 0.010646 2024-04-10T03:06:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:25 smithi152 ceph-mon[33523]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:06:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:25 smithi102 ceph-mon[29301]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:06:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:25 smithi102 ceph-mon[39651]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:06:25.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 77 16 115109 115093 93.3989 56.5 0.0182488 0.0107019 2024-04-10T03:06:26.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 78 16 116123 116107 93.0139 63.375 0.0205089 0.0107468 2024-04-10T03:06:27.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:27 smithi152 ceph-mon[33523]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:27.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:27 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/553027893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:27.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:27 smithi102 ceph-mon[29301]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:27.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:27 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/553027893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:27.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:27 smithi102 ceph-mon[39651]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:27.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:27 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/553027893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:27.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 79 16 117166 117150 92.6617 65.1875 0.00795801 0.0107871 2024-04-10T03:06:28.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:06:28.978398+0000 min lat: 0.00111054 max lat: 0.157392 avg lat: 0.010829 2024-04-10T03:06:28.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:06:28.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 80 16 118101 118085 92.2339 58.4375 0.0315226 0.010829 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: pgmap v134: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: Upgrade: Updating mgr.x 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-mon[33523]: Deploying daemon mgr.x on smithi152 2024-04-10T03:06:29.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[35382]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:06:29.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: pgmap v134: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: Upgrade: Updating mgr.x 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[29301]: Deploying daemon mgr.x on smithi152 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: pgmap v134: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: Upgrade: Updating mgr.x 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:06:29.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:06:29.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:29.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:29 smithi102 ceph-mon[39651]: Deploying daemon mgr.x on smithi152 2024-04-10T03:06:29.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 81 16 119021 119005 91.805 57.5 0.00772852 0.0108881 2024-04-10T03:06:30.374 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/417755012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:30.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/417755012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:30.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/417755012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:30.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 82 16 120051 120035 91.4705 64.375 0.0170935 0.0109255 2024-04-10T03:06:31.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:31 smithi152 ceph-mon[33523]: pgmap v135: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2024-04-10T03:06:31.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:31 smithi102 ceph-mon[39651]: pgmap v135: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2024-04-10T03:06:31.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:31 smithi102 ceph-mon[29301]: pgmap v135: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s 2024-04-10T03:06:31.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 83 16 121030 121014 91.1056 61.1875 0.0265322 0.0109712 2024-04-10T03:06:32.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:32 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3945477784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:32.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:32 smithi152 systemd[1]: Stopping Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:06:32.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:32 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3945477784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:32.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:32] "GET /metrics HTTP/1.1" 200 242759 "" "Prometheus/2.43.0" 2024-04-10T03:06:32.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:32 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3945477784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:32.887 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:32 smithi152 podman[66707]: 2024-04-10 03:06:32.584755199 +0000 UTC m=+0.191261873 container died a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-10T03:06:32.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 84 16 122003 121987 90.7449 60.8125 0.0181533 0.0110151 2024-04-10T03:06:33.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 podman[66707]: 2024-04-10 03:06:33.105038642 +0000 UTC m=+0.711545314 container cleanup a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, ceph=True, org.label-schema.license=GPLv2) 2024-04-10T03:06:33.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 bash[66707]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x 2024-04-10T03:06:33.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:33 smithi102 ceph-mon[29301]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-10T03:06:33.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:33 smithi102 ceph-mon[39651]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-10T03:06:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:33 smithi152 ceph-mon[33523]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-04-10T03:06:33.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 podman[66718]: 2024-04-10 03:06:33.390337086 +0000 UTC m=+0.802288618 container remove a1843f19df0f78e461b1d4de75d1b1d3e3c8115633ca63afcef48b7bd1de2216 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:06:33.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:06:33.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 85 16 122966 122950 90.3854 60.1875 0.0139619 0.0110584 2024-04-10T03:06:34.115 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-10T03:06:34.115 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 systemd[1]: Stopped Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:06:34.115 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:33 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Consumed 18.209s CPU time. 2024-04-10T03:06:34.490 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:34 smithi152 systemd[1]: Starting Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:06:34.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:34 smithi152 podman[66832]: 2024-04-10 03:06:34.396920929 +0000 UTC m=+0.047963862 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:06:34.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:34 smithi152 podman[66832]: 2024-04-10 03:06:34.592101166 +0000 UTC m=+0.243144064 container create da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True) 2024-04-10T03:06:34.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 86 16 123762 123746 89.9129 49.75 0.00957622 0.0111165 2024-04-10T03:06:35.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:35 smithi102 ceph-mon[39651]: pgmap v137: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-10T03:06:35.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:35 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/929428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:35.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:35 smithi102 ceph-mon[29301]: pgmap v137: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-10T03:06:35.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:35 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/929428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:35.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:35 smithi152 ceph-mon[33523]: pgmap v137: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-10T03:06:35.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:35 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/929428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:35.975 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 87 16 124650 124634 89.5173 55.5 0.00483188 0.0111633 2024-04-10T03:06:36.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:36 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:36 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:36 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:36.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 88 16 125577 125561 89.1585 57.9375 0.0377442 0.0112112 2024-04-10T03:06:37.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:36 smithi152 podman[66832]: 2024-04-10 03:06:36.64656393 +0000 UTC m=+2.297606835 container init da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-04-10T03:06:37.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:36 smithi152 podman[66832]: 2024-04-10 03:06:36.658064215 +0000 UTC m=+2.309107116 container start da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-10T03:06:37.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:37 smithi152 ceph-mon[33523]: pgmap v138: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:37.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:37 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1281466208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:37.596 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:37 smithi152 bash[66832]: da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc 2024-04-10T03:06:37.596 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:37 smithi152 systemd[1]: Started Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:06:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:37 smithi102 ceph-mon[29301]: pgmap v138: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:37.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:37 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1281466208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:37.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:37 smithi102 ceph-mon[39651]: pgmap v138: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s 2024-04-10T03:06:37.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:37 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1281466208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:37.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 89 16 126394 126378 88.7305 51.0625 0.0202412 0.0112644 2024-04-10T03:06:38.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:37.597+0000 7f8c76743200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:06:38.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:37.765+0000 7f8c76743200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:06:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[39651]: pgmap v139: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:38 smithi102 ceph-mon[29301]: pgmap v139: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-10T03:06:38.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:38 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:38 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:38.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:38 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:38.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:38 smithi152 ceph-mon[33523]: pgmap v139: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-10T03:06:38.920 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:38.915+0000 7f8c76743200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total time run: 90.0087 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total writes made: 127198 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Write size: 65536 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object size: 65536 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Bandwidth (MB/sec): 88.3234 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Bandwidth: 39.5272 2024-04-10T03:06:38.976 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max bandwidth (MB/sec): 239.438 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min bandwidth (MB/sec): 49.75 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average IOPS: 1413 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev IOPS: 632.435 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max IOPS: 3831 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min IOPS: 796 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average Latency(s): 0.011318 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Latency(s): 0.00847107 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max latency(s): 0.157392 2024-04-10T03:06:38.977 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min latency(s): 0.00111054 2024-04-10T03:06:38.981 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Cleaning up (deleting benchmark objects) 2024-04-10T03:06:39.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:39.059+0000 7f8c76743200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:06:39.577 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:39.283+0000 7f8c76743200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:06:39.577 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:39.439+0000 7f8c76743200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:06:39.862 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:39.578+0000 7f8c76743200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:06:40.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:39 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1358304782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:39 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1358304782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:40.172 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1358304782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:40.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:39.985+0000 7f8c76743200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:06:40.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:40.172+0000 7f8c76743200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:06:40.461 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:40.302+0000 7f8c76743200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:06:40.461 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:40.463+0000 7f8c76743200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:06:40.753 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:40.751+0000 7f8c76743200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:06:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:40 smithi152 ceph-mon[33523]: pgmap v140: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2024-04-10T03:06:41.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:40 smithi102 ceph-mon[29301]: pgmap v140: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2024-04-10T03:06:41.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:40 smithi102 ceph-mon[39651]: pgmap v140: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2024-04-10T03:06:42.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:42 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:42 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:42 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:42 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:42] "GET /metrics HTTP/1.1" 200 242762 "" "Prometheus/2.43.0" 2024-04-10T03:06:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:42 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:42 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:42.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:42.303+0000 7f8c76743200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:06:42.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:42.517+0000 7f8c76743200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:06:43.049 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:43.051+0000 7f8c76743200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:06:43.300 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:43.201+0000 7f8c76743200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:06:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-mon[33523]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-10T03:06:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3321842295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:43.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:43.537+0000 7f8c76743200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:06:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:43 smithi102 ceph-mon[29301]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-10T03:06:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:43 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3321842295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:43 smithi102 ceph-mon[39651]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-10T03:06:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:43 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3321842295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:44.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:43.847+0000 7f8c76743200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:06:44.724 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:44.320+0000 7f8c76743200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:06:45.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:44.726+0000 7f8c76743200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:06:45.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:45 smithi152 ceph-mon[33523]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 55 MiB/s wr, 1.11k op/s 2024-04-10T03:06:45.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:45 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/13428709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:45.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:45 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:45.129+0000 7f8c76743200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:06:45.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:45 smithi102 ceph-mon[29301]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 55 MiB/s wr, 1.11k op/s 2024-04-10T03:06:45.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:45 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/13428709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:45 smithi102 ceph-mon[39651]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 55 MiB/s wr, 1.11k op/s 2024-04-10T03:06:45.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:45 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/13428709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:46.761 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:46.445+0000 7f8c76743200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:06:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:47 smithi152 ceph-mon[33523]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 726 op/s 2024-04-10T03:06:47.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:47.332+0000 7f8c76743200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:06:47.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:47.508+0000 7f8c76743200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:06:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:47 smithi102 ceph-mon[29301]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 726 op/s 2024-04-10T03:06:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:47 smithi102 ceph-mon[39651]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 726 op/s 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:47.883+0000 7f8c76743200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:47] ENGINE Bus STARTING 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: CherryPy Checker: 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: The Application mounted at '' has an empty config. 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:48.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:48] ENGINE Serving on http://:::9283 2024-04-10T03:06:48.263 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:48] ENGINE Bus STARTED 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1278980611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: Standby manager daemon x restarted 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:06:48.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:06:48.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:48 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:06:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1278980611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: Standby manager daemon x restarted 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: Standby manager daemon x started 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1278980611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: Standby manager daemon x restarted 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:06:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:06:48.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:48 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:06:49.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:49 smithi152 ceph-mon[33523]: pgmap v144: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.08k op/s 2024-04-10T03:06:49.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:49 smithi152 ceph-mon[33523]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-10T03:06:49.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:06:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:49 smithi102 ceph-mon[29301]: pgmap v144: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.08k op/s 2024-04-10T03:06:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:49 smithi102 ceph-mon[29301]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-10T03:06:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:49 smithi102 ceph-mon[39651]: pgmap v144: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.08k op/s 2024-04-10T03:06:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:49 smithi102 ceph-mon[39651]: mgrmap e22: y(active, since 4m), standbys: x 2024-04-10T03:06:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:50 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2102277049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:50 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2102277049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:50 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2102277049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:51 smithi152 ceph-mon[33523]: pgmap v145: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 702 op/s 2024-04-10T03:06:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:51 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:51.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:51 smithi102 ceph-mon[29301]: pgmap v145: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 702 op/s 2024-04-10T03:06:51.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:51 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:51.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:51 smithi102 ceph-mon[39651]: pgmap v145: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 702 op/s 2024-04-10T03:06:51.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:51 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:52.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:52] "GET /metrics HTTP/1.1" 200 242843 "" "Prometheus/2.43.0" 2024-04-10T03:06:53.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[29301]: pgmap v146: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 727 op/s 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1018794613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[39651]: pgmap v146: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 727 op/s 2024-04-10T03:06:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:53 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1018794613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:53.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:53 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:53 smithi152 ceph-mon[33523]: pgmap v146: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 727 op/s 2024-04-10T03:06:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:53 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1018794613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:54 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:54.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:54 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:54.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:54 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:54.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:54 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:54.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:54 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:54.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:54 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:55.378 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:55 smithi152 ceph-mon[33523]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.2 MiB/s wr, 1.06k op/s 2024-04-10T03:06:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:55 smithi102 ceph-mon[39651]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.2 MiB/s wr, 1.06k op/s 2024-04-10T03:06:55.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:55 smithi102 ceph-mon[29301]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.2 MiB/s wr, 1.06k op/s 2024-04-10T03:06:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:56 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1836146669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:56.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:56 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1836146669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:56 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1836146669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:57.387 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:57 smithi152 ceph-mon[33523]: pgmap v148: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:06:57.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:57 smithi102 ceph-mon[39651]: pgmap v148: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:06:57.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:57 smithi102 ceph-mon[29301]: pgmap v148: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE Bus STOPPING 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE Bus STOPPED 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE Bus STARTING 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:06:58.494+0000 7fc45ad99700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setuser ceph since I am not root 2024-04-10T03:06:58.781 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ignoring --setgroup ceph since I am not root 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE Serving on http://:::9283 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:06:58] ENGINE Bus STARTED 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.950+0000 7f8bd63dd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.950+0000 7f8bd63dd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.950+0000 7f8bd63dd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.950+0000 7f8bd63dd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.171 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.950+0000 7f8bd63dd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.952+0000 7f8bd33d7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.952+0000 7f8bd33d7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.952+0000 7f8bd33d7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.952+0000 7f8bd33d7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:06:58.952+0000 7f8bd33d7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:06:59.172 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:06:59] "GET /metrics HTTP/1.1" 200 33212 "" "Prometheus/2.43.0" 2024-04-10T03:06:59.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2177527304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:59.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: osdmap e75: 8 total, 8 up, 8 in 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: mgrmap e23: x(active, starting, since 0.0131424s) 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:06:59.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: Manager daemon x is now available 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:06:59.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:06:59 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:06:59.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2177527304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:59.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: osdmap e75: 8 total, 8 up, 8 in 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: mgrmap e23: x(active, starting, since 0.0131424s) 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:06:59.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: Manager daemon x is now available 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:06:59.277+0000 7fd7ed7141c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:06:59.414+0000 7fd7ed7141c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2177527304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.545 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: osdmap e75: 8 total, 8 up, 8 in 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24332 172.21.15.102:0/1910823095' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: mgrmap e23: x(active, starting, since 0.0131424s) 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:06:59.546 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: Manager daemon x is now available 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:06:59.548 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:06:59 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-10T03:07:00.611 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:00.330+0000 7fd7ed7141c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:07:00.611 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:00.480+0000 7fd7ed7141c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:07:00.611 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[39651]: Deploying cephadm binary to smithi102 2024-04-10T03:07:00.611 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[39651]: Deploying cephadm binary to smithi152 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[39651]: mgrmap e24: x(active, since 1.01157s) 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[39651]: pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[29301]: Deploying cephadm binary to smithi102 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[29301]: Deploying cephadm binary to smithi152 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[29301]: mgrmap e24: x(active, since 1.01157s) 2024-04-10T03:07:00.612 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-mon[29301]: pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:00 smithi152 ceph-mon[33523]: Deploying cephadm binary to smithi102 2024-04-10T03:07:00.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:00 smithi152 ceph-mon[33523]: Deploying cephadm binary to smithi152 2024-04-10T03:07:00.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:00 smithi152 ceph-mon[33523]: mgrmap e24: x(active, since 1.01157s) 2024-04-10T03:07:00.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:00 smithi152 ceph-mon[33523]: pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:00.954 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:00.613+0000 7fd7ed7141c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:07:00.954 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:00.833+0000 7fd7ed7141c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:07:00.954 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:00.956+0000 7fd7ed7141c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:07:01.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:01.211+0000 7fd7ed7141c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:07:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:01 smithi152 ceph-mon[33523]: mgrmap e25: x(active, since 2s) 2024-04-10T03:07:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:01 smithi152 ceph-mon[33523]: pgmap v4: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:01 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/727831378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:01 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:01.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:01 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[29301]: mgrmap e25: x(active, since 2s) 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[29301]: pgmap v4: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/727831378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:01.464+0000 7fd7ed7141c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:01.582+0000 7fd7ed7141c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:07:01.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:01.710+0000 7fd7ed7141c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:07:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[39651]: mgrmap e25: x(active, since 2s) 2024-04-10T03:07:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[39651]: pgmap v4: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/727831378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:01.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:02.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:01.835+0000 7fd7ed7141c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:07:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-mon[39651]: pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-mon[39651]: mgrmap e26: x(active, since 4s) 2024-04-10T03:07:03.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:03.405+0000 7fd7ed7141c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:07:03.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:03.610+0000 7fd7ed7141c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:07:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-mon[29301]: pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-mon[29301]: mgrmap e26: x(active, since 4s) 2024-04-10T03:07:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:03 smithi152 ceph-mon[33523]: pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:03 smithi152 ceph-mon[33523]: mgrmap e26: x(active, since 4s) 2024-04-10T03:07:04.241 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:03.925+0000 7fd7ed7141c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:07:04.500 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:04.243+0000 7fd7ed7141c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:07:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:04 smithi152 ceph-mon[33523]: [10/Apr/2024:03:07:03] ENGINE Bus STARTING 2024-04-10T03:07:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:04 smithi152 ceph-mon[33523]: [10/Apr/2024:03:07:03] ENGINE Serving on https://172.21.15.152:7150 2024-04-10T03:07:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:04 smithi152 ceph-mon[33523]: [10/Apr/2024:03:07:03] ENGINE Serving on http://172.21.15.152:8765 2024-04-10T03:07:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:04 smithi152 ceph-mon[33523]: [10/Apr/2024:03:07:03] ENGINE Bus STARTED 2024-04-10T03:07:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:04 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1626446955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:04.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[29301]: [10/Apr/2024:03:07:03] ENGINE Bus STARTING 2024-04-10T03:07:04.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[29301]: [10/Apr/2024:03:07:03] ENGINE Serving on https://172.21.15.152:7150 2024-04-10T03:07:04.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[29301]: [10/Apr/2024:03:07:03] ENGINE Serving on http://172.21.15.152:8765 2024-04-10T03:07:04.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[29301]: [10/Apr/2024:03:07:03] ENGINE Bus STARTED 2024-04-10T03:07:04.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1626446955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:04.502+0000 7fd7ed7141c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:04.769+0000 7fd7ed7141c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[39651]: [10/Apr/2024:03:07:03] ENGINE Bus STARTING 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[39651]: [10/Apr/2024:03:07:03] ENGINE Serving on https://172.21.15.152:7150 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[39651]: [10/Apr/2024:03:07:03] ENGINE Serving on http://172.21.15.152:8765 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[39651]: [10/Apr/2024:03:07:03] ENGINE Bus STARTED 2024-04-10T03:07:04.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:04 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1626446955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:05.532 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:05 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:05.156+0000 7fd7ed7141c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:07:05.532 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:05 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:05.280+0000 7fd7ed7141c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:07:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:05 smithi102 ceph-mon[39651]: pgmap v6: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:05.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:05 smithi102 ceph-mon[29301]: pgmap v6: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:05.935 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:05 smithi152 ceph-mon[33523]: pgmap v6: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-04-10T03:07:06.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:05 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:05.827+0000 7fd7ed7141c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:07:06.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:06 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3018785300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:06.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3018785300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:06 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3018785300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:07.524 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:07.066+0000 7fd7ed7141c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:07:07.524 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:07.525+0000 7fd7ed7141c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:07:07.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:07 smithi102 ceph-mon[29301]: pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-10T03:07:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:07 smithi102 ceph-mon[39651]: pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-10T03:07:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:07 smithi152 ceph-mon[33523]: pgmap v7: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 561 op/s 2024-04-10T03:07:08.258 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:07.968+0000 7fd7ed7141c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:07:08.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:08.260+0000 7fd7ed7141c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:07:08] ENGINE Bus STARTING 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: CherryPy Checker: 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: The Application mounted at '' has an empty config. 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:07:08] ENGINE Serving on http://:::9283 2024-04-10T03:07:08.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: [10/Apr/2024:03:07:08] ENGINE Bus STARTED 2024-04-10T03:07:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:08 smithi152 ceph-mon[33523]: Standby manager daemon y started 2024-04-10T03:07:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:08 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:07:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:08 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:07:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:08 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:07:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:08 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:07:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[29301]: Standby manager daemon y started 2024-04-10T03:07:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:07:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:07:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[39651]: Standby manager daemon y started 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:07:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:08 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/2691584446' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:07:09.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:09] "GET /metrics HTTP/1.1" 200 33212 "" "Prometheus/2.43.0" 2024-04-10T03:07:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:09 smithi152 ceph-mon[33523]: pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-10T03:07:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:09 smithi152 ceph-mon[33523]: mgrmap e27: x(active, since 10s), standbys: y 2024-04-10T03:07:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:09 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:07:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:09 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2448774768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[39651]: pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-10T03:07:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[39651]: mgrmap e27: x(active, since 10s), standbys: y 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2448774768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[29301]: pgmap v8: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 436 op/s 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[29301]: mgrmap e27: x(active, since 10s), standbys: y 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:07:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:09 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2448774768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:11.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:11 smithi102 ceph-mon[39651]: pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 717 op/s 2024-04-10T03:07:11.907 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:11 smithi102 ceph-mon[29301]: pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 717 op/s 2024-04-10T03:07:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:11 smithi152 ceph-mon[33523]: pgmap v9: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 717 op/s 2024-04-10T03:07:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:12 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1565949782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:12 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1565949782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:12.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:07:12.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:12 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1565949782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:13 smithi152 ceph-mon[33523]: pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:13 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:13 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[29301]: pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[39651]: pgmap v10: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:13 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:14 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:14 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3254062576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:14 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:14 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3254062576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:15.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:14 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:15.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:14 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3254062576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:15 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 656 op/s 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:15 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:16 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi102 to 2480M 2024-04-10T03:07:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:16 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi102 to 2480M 2024-04-10T03:07:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:16 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi102 to 2480M 2024-04-10T03:07:17.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:17 smithi152 ceph-mon[33523]: pgmap v12: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 998 op/s 2024-04-10T03:07:17.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:17 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:17.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:17 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:17.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:17 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1011219564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[29301]: pgmap v12: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 998 op/s 2024-04-10T03:07:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1011219564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[39651]: pgmap v12: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 998 op/s 2024-04-10T03:07:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:17 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1011219564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:19.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:19] "GET /metrics HTTP/1.1" 200 36156 "" "Prometheus/2.43.0" 2024-04-10T03:07:19.817 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:19 smithi152 ceph-mon[33523]: pgmap v13: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:07:20.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:19 smithi102 ceph-mon[29301]: pgmap v13: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:07:20.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:19 smithi102 ceph-mon[39651]: pgmap v13: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:07:21.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:20 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3546468977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:21.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:20 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3546468977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:21.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:20 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3546468977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:21.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:21 smithi152 ceph-mon[33523]: pgmap v14: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:22.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:21 smithi102 ceph-mon[39651]: pgmap v14: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:22.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:21 smithi102 ceph-mon[29301]: pgmap v14: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:22 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3205693373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:22 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3205693373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:22.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:07:22.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:22 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3205693373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: pgmap v15: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:07:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:23 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:07:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: pgmap v15: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: pgmap v15: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-10T03:07:24.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:24.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:07:24.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:23 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:07:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:24 smithi152 ceph-mon[33523]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-10T03:07:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:24 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:07:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:24 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:07:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:24 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:24 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[39651]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-10T03:07:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[29301]: Adjusting osd_memory_target on smithi152 to 2480M 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:24 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3572216676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:25.948 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.949 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.950 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3572216676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:25.950 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.950 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.950 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:25.950 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3572216676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:26 smithi152 ceph-mon[33523]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:07:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:26 smithi152 ceph-mon[33523]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:07:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:26 smithi102 ceph-mon[29301]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:07:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:26 smithi102 ceph-mon[29301]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:07:27.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:26 smithi102 ceph-mon[39651]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-10T03:07:27.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:26 smithi102 ceph-mon[39651]: Reconfiguring daemon alertmanager.a on smithi102 2024-04-10T03:07:27.992 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:27 smithi102 ceph-mon[29301]: pgmap v17: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:27.992 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:27 smithi102 ceph-mon[39651]: pgmap v17: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:27 smithi152 ceph-mon[33523]: pgmap v17: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:28 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1771393416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:28 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:29.030 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:28 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1771393416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:29.030 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:28 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:29.030 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:28 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1771393416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:29.030 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:28 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:29.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:29] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.43.0" 2024-04-10T03:07:29.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:29 smithi102 ceph-mon[39651]: pgmap v18: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-10T03:07:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:29 smithi102 ceph-mon[29301]: pgmap v18: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-10T03:07:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:29 smithi152 ceph-mon[33523]: pgmap v18: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-10T03:07:30.694 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:30 smithi102 systemd[1]: Stopping Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:30 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1160726627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:31.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:30 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1160726627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:31.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:30 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1160726627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:31.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[72453]: ts=2024-04-10T03:07:30.822Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:07:31.042 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:30 smithi102 podman[83698]: 2024-04-10 03:07:30.855224227 +0000 UTC m=+0.177801359 container died 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:31.541 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:31 smithi102 podman[83698]: 2024-04-10 03:07:31.414602922 +0000 UTC m=+0.737180059 container cleanup 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:31.541 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:31 smithi102 bash[83698]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a 2024-04-10T03:07:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:31 smithi152 ceph-mon[33523]: pgmap v19: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:07:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:31 smithi102 ceph-mon[29301]: pgmap v19: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:07:32.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:31 smithi102 ceph-mon[39651]: pgmap v19: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:07:32.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:07:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[29538]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:07:32.792 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 podman[83708]: 2024-04-10 03:07:32.30883464 +0000 UTC m=+1.444561751 container remove 4f5410539eed2374ff830c143b6516be94a1b416edb145455fd55e4cccdc845c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:32.792 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 podman[83708]: 2024-04-10 03:07:32.415785674 +0000 UTC m=+1.551512793 volume remove 7b9e94ffdc9fd282d5aeb5be05ba29cc2459fc2acee04ffb8bd397a0114d1811 2024-04-10T03:07:33.183 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-10T03:07:33.183 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 systemd[1]: Stopped Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:33.183 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service: Consumed 1.273s CPU time. 2024-04-10T03:07:33.183 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:32 smithi102 systemd[1]: Starting Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:33.442 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 podman[83811]: 2024-04-10 03:07:33.086650955 +0000 UTC m=+0.036594274 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-10T03:07:33.697 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 podman[83811]: 2024-04-10 03:07:33.444983285 +0000 UTC m=+0.394926590 volume create 0da420fff6cab3c90be1b37baa9e0e8ba889f9e7127053736066ad67ad3c4883 2024-04-10T03:07:33.697 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 podman[83811]: 2024-04-10 03:07:33.570934282 +0000 UTC m=+0.520877584 container create 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:33.948 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-mon[39651]: pgmap v20: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:33.948 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4121129193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:33.948 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 podman[83811]: 2024-04-10 03:07:33.925072284 +0000 UTC m=+0.875015591 container init 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:33.949 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 podman[83811]: 2024-04-10 03:07:33.930495983 +0000 UTC m=+0.880439293 container start 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:33.949 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-mon[29301]: pgmap v20: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:33.949 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4121129193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:33 smithi152 ceph-mon[33523]: pgmap v20: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:33 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4121129193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.951Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.951Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.952Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.102 port=9094 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.957Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.991Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:33.992Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:34.000Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:34.000Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:34 smithi102 bash[83811]: 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 2024-04-10T03:07:34.292 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:34 smithi102 systemd[1]: Started Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[39651]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[39651]: Reconfiguring daemon node-exporter.a on smithi102 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[39651]: pgmap v21: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:35.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[29301]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-10T03:07:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[29301]: Reconfiguring daemon node-exporter.a on smithi102 2024-04-10T03:07:35.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-mon[29301]: pgmap v21: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:35.293 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:34 smithi102 systemd[1]: Stopping Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:35.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:35 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:35 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:35.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:35 smithi152 ceph-mon[33523]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-10T03:07:35.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:35 smithi152 ceph-mon[33523]: Reconfiguring daemon node-exporter.a on smithi102 2024-04-10T03:07:35.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:35 smithi152 ceph-mon[33523]: pgmap v21: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-10T03:07:35.710 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:35 smithi102 podman[83983]: 2024-04-10 03:07:35.474214435 +0000 UTC m=+0.160303904 container died 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:35.986 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:35 smithi102 podman[83983]: 2024-04-10 03:07:35.989228146 +0000 UTC m=+0.675317606 container cleanup 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:35.986 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:35 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:35.957Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000244368s 2024-04-10T03:07:36.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:36 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1838859564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:36.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:36 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1838859564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:36.292 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:35 smithi102 bash[83983]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a 2024-04-10T03:07:36.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:36 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1838859564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:36.613 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 podman[83997]: 2024-04-10 03:07:36.366488525 +0000 UTC m=+0.891789728 container remove 02931ad901dc13d64100edda4dae6da79d9cb58af6ed17c9bf5e7a1ad0706935 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:36.614 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:07:36.614 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-10T03:07:36.614 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 systemd[1]: Stopped Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:36.883 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Consumed 2.430s CPU time. 2024-04-10T03:07:36.883 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 systemd[1]: Starting Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:37.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-mon[29301]: pgmap v22: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:37.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-mon[39651]: pgmap v22: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:37.292 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 podman[84080]: 2024-04-10 03:07:36.791702052 +0000 UTC m=+0.027085343 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-10T03:07:37.292 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:36 smithi102 podman[84080]: 2024-04-10 03:07:36.926429919 +0000 UTC m=+0.161813203 container create 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:37.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:37 smithi152 ceph-mon[33523]: pgmap v22: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 podman[84080]: 2024-04-10 03:07:37.314413014 +0000 UTC m=+0.549796304 container init 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 podman[84080]: 2024-04-10 03:07:37.319933522 +0000 UTC m=+0.555316815 container start 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.324Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.324Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.325Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.325Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.326Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-10T03:07:37.792 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.326Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=arp 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-10T03:07:37.793 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=edac 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-10T03:07:37.794 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=os 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-10T03:07:37.795 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=stat 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=time 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=uname 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a[84092]: ts=2024-04-10T03:07:37.327Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-10T03:07:37.796 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 bash[84080]: 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 2024-04-10T03:07:37.797 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:07:37 smithi102 systemd[1]: Started Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:38.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:38 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:38 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:38 smithi152 ceph-mon[33523]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:07:38.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:38 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1152044390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[39651]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1152044390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[29301]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:07:38.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:38 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1152044390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:39.478 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:39] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.43.0" 2024-04-10T03:07:39.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:39 smithi152 ceph-mon[33523]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:07:39.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:39 smithi152 ceph-mon[33523]: pgmap v23: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:07:39.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:39 smithi102 ceph-mon[29301]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:07:39.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:39 smithi102 ceph-mon[29301]: pgmap v23: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:07:39.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:39 smithi102 ceph-mon[39651]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:07:39.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:39 smithi102 ceph-mon[39651]: pgmap v23: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:07:41.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:41 smithi152 ceph-mon[33523]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:07:41.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:41 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4080459035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:41 smithi102 ceph-mon[39651]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:07:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:41 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4080459035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:41 smithi102 ceph-mon[29301]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:07:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:41 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4080459035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:42.261 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:41 smithi152 systemd[1]: Stopping Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.369Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.370Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-10T03:07:42.682 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.371Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-10T03:07:42.683 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.371Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-10T03:07:42.683 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[64077]: ts=2024-04-10T03:07:42.371Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-10T03:07:42.683 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 podman[74797]: 2024-04-10 03:07:42.380211855 +0000 UTC m=+0.161758352 container died a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:43.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 podman[74797]: 2024-04-10 03:07:42.774715571 +0000 UTC m=+0.556262063 container cleanup a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:43.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:42 smithi152 bash[74797]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a 2024-04-10T03:07:43.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 podman[74806]: 2024-04-10 03:07:43.051453486 +0000 UTC m=+0.665835081 container remove a2d3e5fc2d74416fdee43608875c96d899dfb590b825ed20e4d943d2898ba54f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:43.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-10T03:07:43.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 systemd[1]: Stopped Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:43.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Consumed 2.022s CPU time. 2024-04-10T03:07:43.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 systemd[1]: Starting Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:43.957 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:43 smithi102 ceph-mon[29301]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:43.958 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:43 smithi102 ceph-mon[39651]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:43.991 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:43 smithi152 ceph-mon[33523]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:43.991 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 podman[74884]: 2024-04-10 03:07:43.625384838 +0000 UTC m=+0.059135770 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-10T03:07:44.253 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:43 smithi152 podman[74884]: 2024-04-10 03:07:43.994451117 +0000 UTC m=+0.428202058 container create 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:44.253 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 podman[74884]: 2024-04-10 03:07:44.256053317 +0000 UTC m=+0.689804231 container init 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:44.291 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:07:43 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:07:43.960Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003273363s 2024-04-10T03:07:44.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 podman[74884]: 2024-04-10 03:07:44.259637225 +0000 UTC m=+0.693388128 container start 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.305Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.305Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.305Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi152 (none))" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.305Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.306Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.308Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.313Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.315Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.315Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.318Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.318Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.438µs 2024-04-10T03:07:44.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.318Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.319Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 bash[74884]: 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 systemd[1]: Started Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.353Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.353Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.353Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.392µs wal_replay_duration=34.520984ms wbl_replay_duration=152ns total_replay_duration=34.57679ms 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.357Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.357Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.357Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.402Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=44.189636ms db_storage=1.289µs remote_storage=1.646µs web_handler=437ns query_engine=1.382µs scrape=378.777µs scrape_sd=127.498µs notify=15.592µs notify_sd=14.893µs rules=42.958237ms tracing=8.06µs 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.402Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-10T03:07:44.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:44.402Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-10T03:07:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3940735923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:44 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3940735923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:45.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3940735923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:44 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:07:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:45 smithi152 ceph-mon[33523]: Reconfiguring grafana.a (dependencies changed)... 2024-04-10T03:07:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:45 smithi152 ceph-mon[33523]: Reconfiguring daemon grafana.a on smithi152 2024-04-10T03:07:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:45 smithi152 ceph-mon[33523]: pgmap v26: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[39651]: Reconfiguring grafana.a (dependencies changed)... 2024-04-10T03:07:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[39651]: Reconfiguring daemon grafana.a on smithi152 2024-04-10T03:07:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[39651]: pgmap v26: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[29301]: Reconfiguring grafana.a (dependencies changed)... 2024-04-10T03:07:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[29301]: Reconfiguring daemon grafana.a on smithi152 2024-04-10T03:07:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:45 smithi102 ceph-mon[29301]: pgmap v26: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:07:46.940 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:46 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/267455072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:46 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/267455072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:46 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/267455072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:48.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:47 smithi152 ceph-mon[33523]: pgmap v27: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:48.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:47 smithi102 ceph-mon[39651]: pgmap v27: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:48.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:47 smithi102 ceph-mon[29301]: pgmap v27: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:07:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:49 smithi152 ceph-mon[33523]: pgmap v28: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-10T03:07:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:49 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2213349589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:50.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:49 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:49] "GET /metrics HTTP/1.1" 200 36148 "" "Prometheus/2.43.0" 2024-04-10T03:07:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:49 smithi102 ceph-mon[39651]: pgmap v28: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-10T03:07:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:49 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2213349589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:49 smithi102 ceph-mon[29301]: pgmap v28: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-10T03:07:50.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:49 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2213349589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:52.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:51 smithi152 ceph-mon[33523]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:52.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:51 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3458220779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:52.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:51 smithi102 ceph-mon[29301]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:51 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3458220779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:51 smithi102 ceph-mon[39651]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:07:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:51 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3458220779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:54.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:53 smithi152 ceph-mon[33523]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:53 smithi102 ceph-mon[39651]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:53 smithi102 ceph-mon[29301]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:54 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/840959611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:54 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/840959611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:54 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/840959611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:55.991 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:55 smithi152 ceph-mon[33523]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:55 smithi102 ceph-mon[29301]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:56.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:55 smithi102 ceph-mon[39651]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-10T03:07:57.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:57 smithi152 ceph-mon[33523]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-10T03:07:57.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:57 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2021126667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:57 smithi102 ceph-mon[39651]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-10T03:07:58.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:57 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2021126667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:57 smithi102 ceph-mon[29301]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-10T03:07:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:57 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2021126667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:58.754 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:07:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:07:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:07:59.011 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:58 smithi152 systemd[1]: Stopping Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:07:59.578 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:59 smithi152 podman[75165]: 2024-04-10 03:07:59.274809468 +0000 UTC m=+0.145638199 container died 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.openshift.expose-services=, description=Ceph Grafana Container, vcs-type=git, version=8.5, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664) 2024-04-10T03:07:59.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:59 smithi152 ceph-mon[33523]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:07:59.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:59 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:07:59.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:07:59 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/81509879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:07:59.971 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:07:59 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:07:59] "GET /metrics HTTP/1.1" 200 36150 "" "Prometheus/2.43.0" 2024-04-10T03:07:59.971 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:59 smithi152 podman[75165]: 2024-04-10 03:07:59.67089565 +0000 UTC m=+0.541724380 container cleanup 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, description=Ceph Grafana Container, version=8.5, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-10T03:07:59.971 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:59 smithi152 bash[75165]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a 2024-04-10T03:08:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[29301]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:08:00.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:00.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/81509879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[39651]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:08:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:07:59 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/81509879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:00.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:59 smithi152 podman[75176]: 2024-04-10 03:07:59.973274813 +0000 UTC m=+0.697780496 container remove 64967280ffeee6018db7bf000e1b8efe787f681564c9cee6d38983231cffb6ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-04-10T03:08:00.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:07:59 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:08:00.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:00.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:00.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 6.116s CPU time. 2024-04-10T03:08:00.263 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:00.818 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 podman[75250]: 2024-04-10 03:08:00.434474326 +0000 UTC m=+0.042413447 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:08:00.818 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 podman[75250]: 2024-04-10 03:08:00.558058085 +0000 UTC m=+0.165997194 container create c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, vcs-type=git, architecture=x86_64, io.buildah.version=1.32.0, distribution-scope=public, name=ubi8, release=239.1651231664) 2024-04-10T03:08:01.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 podman[75250]: 2024-04-10 03:08:00.819687825 +0000 UTC m=+0.427626932 container init c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, distribution-scope=public, version=8.5, maintainer=Paul Cuzner , architecture=x86_64, description=Ceph Grafana Container, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-10T03:08:01.077 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 podman[75250]: 2024-04-10 03:08:00.824193867 +0000 UTC m=+0.432132987 container start c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, name=ubi8, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container, release=239.1651231664, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5) 2024-04-10T03:08:01.078 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 bash[75250]: c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 2024-04-10T03:08:01.078 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:00 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:01.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:01 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[75261]: logger=settings t=2024-04-10T03:08:01.14541089Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:08:01.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:01 smithi152 podman[75331]: 2024-04-10 03:08:01.203527328 +0000 UTC m=+0.040124958 container died c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, version=8.5, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.expose-services=) 2024-04-10T03:08:01.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:01 smithi152 ceph-mon[33523]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-04-10T03:08:01.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:01 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:01.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:01 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:02.011 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:01 smithi152 systemd[1]: Stopping Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:02.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:01 smithi152 podman[75331]: 2024-04-10 03:08:01.835247442 +0000 UTC m=+0.671845073 container remove c0630fcd2fa65c3424c249e663589380caff1b69a6f2501c541ef3e404e28af3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, name=ubi8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, distribution-scope=public, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc.) 2024-04-10T03:08:02.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:01 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:08:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[29301]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-04-10T03:08:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:02.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[39651]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-04-10T03:08:02.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:02.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:01 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:02.668 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:02 smithi152 podman[75439]: 2024-04-10 03:08:02.451782621 +0000 UTC m=+0.165281194 container died 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:02.668 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:02 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:02.668 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:02 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.027s CPU time. 2024-04-10T03:08:02.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:02 smithi152 ceph-mon[33523]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-10T03:08:02.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:02 smithi152 ceph-mon[33523]: Reconfiguring daemon node-exporter.b on smithi152 2024-04-10T03:08:02.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:02 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1399546255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[29301]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-10T03:08:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[29301]: Reconfiguring daemon node-exporter.b on smithi152 2024-04-10T03:08:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1399546255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[39651]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-10T03:08:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[39651]: Reconfiguring daemon node-exporter.b on smithi152 2024-04-10T03:08:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:02 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1399546255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:03.562 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 podman[75439]: 2024-04-10 03:08:03.271436049 +0000 UTC m=+0.984934621 container cleanup 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:03.562 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 bash[75439]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b 2024-04-10T03:08:03.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:03 smithi102 ceph-mon[29301]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:03.942 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:03 smithi102 ceph-mon[39651]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:03 smithi152 ceph-mon[33523]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 podman[75451]: 2024-04-10 03:08:03.565312095 +0000 UTC m=+1.114667000 container remove 6f30ab34e787f033b234b0626573597c936e1d69e20160a211751aceab35ca87 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 systemd[1]: Stopped Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Consumed 2.497s CPU time. 2024-04-10T03:08:04.012 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:03 smithi152 systemd[1]: Starting Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:04.295 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 podman[75527]: 2024-04-10 03:08:04.040547085 +0000 UTC m=+0.032720843 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-10T03:08:04.295 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 podman[75527]: 2024-04-10 03:08:04.166733509 +0000 UTC m=+0.158907263 container create 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:04.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 podman[75527]: 2024-04-10 03:08:04.403647773 +0000 UTC m=+0.395821527 container init 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:04.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 podman[75527]: 2024-04-10 03:08:04.40751855 +0000 UTC m=+0.399692302 container start 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:08:04.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.414Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-10T03:08:04.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.414Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-10T03:08:04.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.415Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.415Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.415Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.416Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.416Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.416Z caller=node_exporter.go:117 level=info collector=arp 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.416Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.416Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-10T03:08:04.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=edac 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-10T03:08:04.764 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=os 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-10T03:08:04.765 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=stat 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.417Z caller=node_exporter.go:117 level=info collector=time 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=node_exporter.go:117 level=info collector=uname 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b[75539]: ts=2024-04-10T03:08:04.418Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 bash[75527]: 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d 2024-04-10T03:08:04.766 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:08:04 smithi152 systemd[1]: Started Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1363078443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:05.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:05.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:05.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:05 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1363078443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:06.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-10T03:08:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:05 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1363078443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:07.856 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-04-10T03:08:07.856 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:07.856 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:07.856 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2623563802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:07.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:07.858 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:07 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2623563802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:08.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:07 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2623563802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:08.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:08 smithi102 ceph-mon[29301]: Upgrade: Updating mgr.y 2024-04-10T03:08:08.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:08 smithi102 ceph-mon[29301]: Deploying daemon mgr.y on smithi102 2024-04-10T03:08:08.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:08 smithi102 ceph-mon[39651]: Upgrade: Updating mgr.y 2024-04-10T03:08:08.858 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:08 smithi102 ceph-mon[39651]: Deploying daemon mgr.y on smithi102 2024-04-10T03:08:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:08 smithi152 ceph-mon[33523]: Upgrade: Updating mgr.y 2024-04-10T03:08:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:08 smithi152 ceph-mon[33523]: Deploying daemon mgr.y on smithi102 2024-04-10T03:08:09.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:08:09.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:09 smithi102 systemd[1]: Stopping Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:09 smithi152 ceph-mon[33523]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-10T03:08:10.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:08:09] "GET /metrics HTTP/1.1" 200 36150 "" "Prometheus/2.43.0" 2024-04-10T03:08:10.039 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:09 smithi102 podman[84693]: 2024-04-10 03:08:09.71380594 +0000 UTC m=+0.210816955 container died 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-10T03:08:10.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:09 smithi102 ceph-mon[39651]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-10T03:08:10.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:09 smithi102 ceph-mon[29301]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-10T03:08:10.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 podman[84693]: 2024-04-10 03:08:10.149566266 +0000 UTC m=+0.646577284 container cleanup 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-10T03:08:10.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 bash[84693]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y 2024-04-10T03:08:10.766 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 podman[84725]: 2024-04-10 03:08:10.41845021 +0000 UTC m=+0.699017231 container remove 8be8627dd2883b7f5f9d40dab46249da37ecfb992b63fea737650f0e94502160 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-10T03:08:10.766 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:08:10.767 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:10 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1179627800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:10.767 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:10 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1179627800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:10 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1179627800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:11.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service: Failed with result 'exit-code'. 2024-04-10T03:08:11.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 systemd[1]: Stopped Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:11.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:10 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service: Consumed 1min 1.581s CPU time. 2024-04-10T03:08:11.389 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 systemd[1]: Starting Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:11.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:11 smithi102 ceph-mon[39651]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:11.679 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:11 smithi102 ceph-mon[29301]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:11.679 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 podman[84844]: 2024-04-10 03:08:11.295833893 +0000 UTC m=+0.038506272 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:08:11.680 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 podman[84844]: 2024-04-10 03:08:11.420234233 +0000 UTC m=+0.162906603 container create ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5) 2024-04-10T03:08:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:11 smithi152 ceph-mon[33523]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:12.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 podman[84844]: 2024-04-10 03:08:11.68221031 +0000 UTC m=+0.424882692 container init ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, GIT_BRANCH=HEAD) 2024-04-10T03:08:12.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 podman[84844]: 2024-04-10 03:08:11.687574016 +0000 UTC m=+0.430246390 container start ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:08:12.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 ceph-mgr[84858]: -- 172.21.15.102:0/2092078087 <== mon.2 v2:172.21.15.102:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ea366925a0 con 0x55ea36666c00 2024-04-10T03:08:12.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 bash[84844]: ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 2024-04-10T03:08:12.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:11 smithi102 systemd[1]: Started Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:12.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:12.650+0000 7f9e1aa42200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:08:12.836 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-10T03:08:12.837 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:12.837 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.027s CPU time. 2024-04-10T03:08:12.837 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:13.094 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.095 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:12 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3137063806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:13.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 podman[75675]: 2024-04-10 03:08:12.744515107 +0000 UTC m=+0.035164249 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:08:13.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:12 smithi152 podman[75675]: 2024-04-10 03:08:12.886126172 +0000 UTC m=+0.176775303 container create 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.component=ubi8-container, name=ubi8, io.buildah.version=1.32.0, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-10T03:08:13.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:12.812+0000 7f9e1aa42200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3137063806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:13.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:12 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3137063806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:13.505 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:13 smithi152 podman[75675]: 2024-04-10 03:08:13.19830085 +0000 UTC m=+0.488949977 container init 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8.5, description=Ceph Grafana Container, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , architecture=x86_64, io.openshift.expose-services=, name=ubi8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:08:13.506 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:13 smithi152 podman[75675]: 2024-04-10 03:08:13.203000046 +0000 UTC m=+0.493649180 container start 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, name=ubi8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897) 2024-04-10T03:08:13.506 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:13 smithi152 bash[75675]: 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 2024-04-10T03:08:13.506 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:13 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:13.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:13 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[75700]: logger=settings t=2024-04-10T03:08:13.508458845Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:08:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:13 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:14.236 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:13 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:14.237 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:13 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:13.931+0000 7f9e1aa42200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:08:14.237 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:14.096+0000 7f9e1aa42200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:08:14.237 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:13 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:14.495 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:14.331+0000 7f9e1aa42200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:08:14.495 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:14.497+0000 7f9e1aa42200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:08:14.511 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:14 smithi152 podman[75752]: 2024-04-10 03:08:14.196830047 +0000 UTC m=+0.673703876 container died 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, name=ubi8, distribution-scope=public, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=) 2024-04-10T03:08:14.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:14.625+0000 7f9e1aa42200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:08:15.203 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:15.089+0000 7f9e1aa42200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:08:15.204 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-mon[29301]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:15.204 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:14 smithi102 ceph-mon[39651]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:15.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:14 smithi152 ceph-mon[33523]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-04-10T03:08:15.489 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:15.318+0000 7f9e1aa42200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:08:15.798 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:15.491+0000 7f9e1aa42200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:08:15.798 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:15.671+0000 7f9e1aa42200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:08:15.938 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:15 smithi152 podman[75752]: 2024-04-10 03:08:15.619147017 +0000 UTC m=+2.096020829 container remove 525b1dd3465abbf598f10f906fbbce9877564c0ba9930be815e832f73d1a8138 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, distribution-scope=public, architecture=x86_64, io.openshift.tags=base rhel8) 2024-04-10T03:08:15.939 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:15 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:08:16.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:15 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3688459402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:16.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3688459402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:16.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:15.970+0000 7f9e1aa42200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:08:16.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:15 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3688459402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:16.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:16 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:16.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:16 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.041s CPU time. 2024-04-10T03:08:17.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:16 smithi152 ceph-mon[33523]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:08:17.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:16 smithi102 ceph-mon[29301]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:08:17.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:16 smithi102 ceph-mon[39651]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:08:17.775 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:17 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:17.550+0000 7f9e1aa42200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:08:18.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:17 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:17.773+0000 7f9e1aa42200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:08:18.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:18.337+0000 7f9e1aa42200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:08:18.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:18.507+0000 7f9e1aa42200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:08:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:18 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:18 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:18 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3755493947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:18 smithi152 ceph-mon[33523]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-10T03:08:19.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3755493947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[29301]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:18.859+0000 7f9e1aa42200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:08:19.173 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:19 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:19.173+0000 7f9e1aa42200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:08:19.174 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.174 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:19.174 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3755493947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:19.174 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:18 smithi102 ceph-mon[39651]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-10T03:08:20.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:08:19] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.43.0" 2024-04-10T03:08:20.036 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:19 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:19.645+0000 7f9e1aa42200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:08:20.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:20.037+0000 7f9e1aa42200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:08:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:20 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:20 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:20 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:20 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:20.738 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:20.458+0000 7f9e1aa42200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:08:21.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:21 smithi102 ceph-mon[29301]: pgmap v44: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:08:21.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:21 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2679405364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:21.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:21 smithi102 ceph-mon[39651]: pgmap v44: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:08:21.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:21 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2679405364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:21.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:21 smithi152 ceph-mon[33523]: pgmap v44: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:08:21.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:21 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2679405364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:22.131 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:21 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:21.830+0000 7f9e1aa42200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:08:23.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:22.772+0000 7f9e1aa42200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:08:23.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:22.937+0000 7f9e1aa42200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:23.331+0000 7f9e1aa42200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:23] ENGINE Bus STARTING 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: CherryPy Checker: 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: The Application mounted at '' has an empty config. 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:23] ENGINE Serving on http://:::9283 2024-04-10T03:08:23.597 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:23] ENGINE Bus STARTED 2024-04-10T03:08:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: pgmap v45: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: Standby manager daemon y restarted 2024-04-10T03:08:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: Standby manager daemon y started 2024-04-10T03:08:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:08:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:08:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:23 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: pgmap v45: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: Standby manager daemon y restarted 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: Standby manager daemon y started 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: pgmap v45: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: Standby manager daemon y restarted 2024-04-10T03:08:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: Standby manager daemon y started 2024-04-10T03:08:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-10T03:08:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-10T03:08:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:23 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:24 smithi102 ceph-mon[29301]: mgrmap e28: x(active, since 85s), standbys: y 2024-04-10T03:08:24.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:24 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1058210041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:24.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:24 smithi102 ceph-mon[39651]: mgrmap e28: x(active, since 85s), standbys: y 2024-04-10T03:08:24.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:24 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1058210041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:24 smithi152 ceph-mon[33523]: mgrmap e28: x(active, since 85s), standbys: y 2024-04-10T03:08:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:24 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1058210041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: pgmap v46: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: pgmap v46: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: pgmap v46: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:25.781+0000 7f8c69d37700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ignoring --setuser ceph since I am not root 2024-04-10T03:08:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ignoring --setgroup ceph since I am not root 2024-04-10T03:08:26.084 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:25] ENGINE Bus STOPPING 2024-04-10T03:08:26.084 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:08:26.084 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:25] ENGINE Bus STOPPED 2024-04-10T03:08:26.085 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:25] ENGINE Bus STARTING 2024-04-10T03:08:26.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-10T03:08:26.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:26.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.041s CPU time. 2024-04-10T03:08:26.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:26] ENGINE Serving on http://:::9283 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:08:26] ENGINE Bus STARTED 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.224+0000 7f9d7aedd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.224+0000 7f9d7aedd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.224+0000 7f9d7aedd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.224+0000 7f9d7aedd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.224+0000 7f9d7aedd700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.225+0000 7f9d77ed7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.225+0000 7f9d77ed7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.225+0000 7f9d77ed7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.225+0000 7f9d77ed7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:08:26.225+0000 7f9d77ed7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:08:26.926 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 podman[76343]: 2024-04-10 03:08:26.554257155 +0000 UTC m=+0.058958008 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:08:26.926 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 podman[76343]: 2024-04-10 03:08:26.697392751 +0000 UTC m=+0.202093582 container create 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, vcs-type=git, description=Ceph Grafana Container, version=8.5, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.24727 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: mgrmap e29: y(active, starting, since 0.013656s) 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:08:26.927 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: Manager daemon y is now available 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:26.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-mon[33523]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:26.760+0000 7f28d901a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:08:26.929 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:26.927+0000 7f28d901a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:08:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.24727 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: mgrmap e29: y(active, starting, since 0.013656s) 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:08:27.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: Manager daemon y is now available 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.24727 172.21.15.152:0/1799933721' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:27.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.24727 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: mgrmap e29: y(active, starting, since 0.013656s) 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:08:27.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: Manager daemon y is now available 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:27.046 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[29301]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:08:27.047 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:26 smithi102 ceph-mon[39651]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:27.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 podman[76343]: 2024-04-10 03:08:26.984308264 +0000 UTC m=+0.489009105 container init 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.openshift.expose-services=, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, maintainer=Paul Cuzner , distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-10T03:08:27.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:26 smithi152 podman[76343]: 2024-04-10 03:08:26.994978062 +0000 UTC m=+0.499678899 container start 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.buildah.version=1.32.0, vcs-type=git, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, distribution-scope=public, release=239.1651231664, com.redhat.component=ubi8-container, version=8.5) 2024-04-10T03:08:27.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:27 smithi152 bash[76343]: 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d 2024-04-10T03:08:27.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:27 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:27.625 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[76374]: logger=settings t=2024-04-10T03:08:27.377061744Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:08:27.625 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:27 smithi152 podman[76442]: 2024-04-10 03:08:27.442721708 +0000 UTC m=+0.047614190 container died 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, distribution-scope=public, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc.) 2024-04-10T03:08:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-mon[33523]: [10/Apr/2024:03:08:26] ENGINE Bus STARTING 2024-04-10T03:08:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2489590481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-mon[33523]: mgrmap e30: y(active, since 1.01154s) 2024-04-10T03:08:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:27 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[29301]: [10/Apr/2024:03:08:26] ENGINE Bus STARTING 2024-04-10T03:08:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2489590481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[29301]: mgrmap e30: y(active, since 1.01154s) 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[39651]: [10/Apr/2024:03:08:26] ENGINE Bus STARTING 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2489590481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[39651]: mgrmap e30: y(active, since 1.01154s) 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:27 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:28.502 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:28.088+0000 7f28d901a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:08:28.502 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:28.256+0000 7f28d901a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:08:28.502 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:28 smithi152 podman[76442]: 2024-04-10 03:08:28.04873886 +0000 UTC m=+0.653631329 container remove 1639ceecc47d34e7e62f0f1f23ef930398b5da14d75b0b1737df4f70a00c615d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., description=Ceph Grafana Container, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, io.buildah.version=1.32.0, io.openshift.expose-services=, io.openshift.tags=base rhel8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, name=ubi8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-10T03:08:28.502 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:28 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:08:28.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:28 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:28.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:28 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.127s CPU time. 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: [10/Apr/2024:03:08:26] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: pgmap v3: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: [10/Apr/2024:03:08:26] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: [10/Apr/2024:03:08:26] ENGINE Bus STARTED 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-mon[33523]: mgrmap e31: y(active, since 2s) 2024-04-10T03:08:28.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:28.503+0000 7f28d901a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:08:28.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:28.675+0000 7f28d901a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:08:28.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:08:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: [10/Apr/2024:03:08:26] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: pgmap v3: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: [10/Apr/2024:03:08:26] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: [10/Apr/2024:03:08:26] ENGINE Bus STARTED 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: Cluster is now healthy 2024-04-10T03:08:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[29301]: mgrmap e31: y(active, since 2s) 2024-04-10T03:08:29.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: [10/Apr/2024:03:08:26] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:08:29.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: pgmap v3: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:29.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: [10/Apr/2024:03:08:26] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:08:29.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: [10/Apr/2024:03:08:26] ENGINE Bus STARTED 2024-04-10T03:08:29.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:08:29.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:08:29.045 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:28 smithi102 ceph-mon[39651]: mgrmap e31: y(active, since 2s) 2024-04-10T03:08:29.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:28.833+0000 7f28d901a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:08:29.608 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:29.270+0000 7f28d901a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:08:29.608 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:29.465+0000 7f28d901a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:08:30.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-mon[33523]: pgmap v4: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:30.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/712626570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:30.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:29.607+0000 7f28d901a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:08:30.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:29.774+0000 7f28d901a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:08:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:29 smithi102 ceph-mon[29301]: pgmap v4: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:29 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/712626570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:30.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:29 smithi102 ceph-mon[39651]: pgmap v4: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:30.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:29 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/712626570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:30.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:30.051+0000 7f28d901a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:08:31.079 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:30 smithi152 ceph-mon[33523]: mgrmap e32: y(active, since 4s) 2024-04-10T03:08:31.079 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:30 smithi152 ceph-mon[33523]: pgmap v5: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:31.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:30 smithi102 ceph-mon[29301]: mgrmap e32: y(active, since 4s) 2024-04-10T03:08:31.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:30 smithi102 ceph-mon[29301]: pgmap v5: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:31.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:30 smithi102 ceph-mon[39651]: mgrmap e32: y(active, since 4s) 2024-04-10T03:08:31.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:30 smithi102 ceph-mon[39651]: pgmap v5: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:32.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:31.641+0000 7f28d901a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:08:32.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:31.863+0000 7f28d901a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:08:32.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:32.410+0000 7f28d901a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:08:32.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:32.562+0000 7f28d901a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:08:33.125 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:32 smithi102 ceph-mon[39651]: pgmap v6: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:33.125 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:32 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1533600349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:33.125 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:32 smithi102 ceph-mon[29301]: pgmap v6: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:33.126 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:32 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1533600349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:33.238 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:32 smithi152 ceph-mon[33523]: pgmap v6: 89 pgs: 89 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail 2024-04-10T03:08:33.238 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:32 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1533600349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:33.238 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:32.917+0000 7f28d901a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:08:33.238 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:33.237+0000 7f28d901a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:08:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:33.760+0000 7f28d901a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:08:34.177 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.177 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.177 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.178 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.178 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.178 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.178 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.178 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:33 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:34.456 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:34.169+0000 7f28d901a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:08:34.727 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:34.601+0000 7f28d901a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:08:35.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 499 op/s 2024-04-10T03:08:35.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3154731062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 499 op/s 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3154731062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:35 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: pgmap v7: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 499 op/s 2024-04-10T03:08:35.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3154731062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:35.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:36.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:35.968+0000 7f28d901a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:08:37.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:36.895+0000 7f28d901a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:08:37.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:37.061+0000 7f28d901a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:08:37.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 388 op/s 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3427050551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:37.428+0000 7f28d901a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:08:37.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:08:37] ENGINE Bus STARTING 2024-04-10T03:08:37.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: CherryPy Checker: 2024-04-10T03:08:37.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: The Application mounted at '' has an empty config. 2024-04-10T03:08:37.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: 2024-04-10T03:08:37.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:08:37] ENGINE Serving on http://:::9283 2024-04-10T03:08:37.763 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: [10/Apr/2024:03:08:37] ENGINE Bus STARTED 2024-04-10T03:08:37.856 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 388 op/s 2024-04-10T03:08:37.856 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3427050551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:37.856 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: pgmap v8: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 388 op/s 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3427050551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: Standby manager daemon x started 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:08:37.857 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:37 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/2174016184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:08:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:38 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:08:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:38 smithi152 ceph-mon[33523]: mgrmap e33: y(active, since 11s), standbys: x 2024-04-10T03:08:39.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:38 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-10T03:08:39.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:38 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:39.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:38 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.127s CPU time. 2024-04-10T03:08:39.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:38 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:38 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:08:39.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:38 smithi102 ceph-mon[29301]: mgrmap e33: y(active, since 11s), standbys: x 2024-04-10T03:08:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:38 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:08:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:38 smithi102 ceph-mon[39651]: mgrmap e33: y(active, since 11s), standbys: x 2024-04-10T03:08:39.400 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 podman[77178]: 2024-04-10 03:08:39.024399764 +0000 UTC m=+0.030382058 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:08:39.400 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 podman[77178]: 2024-04-10 03:08:39.149540304 +0000 UTC m=+0.155522579 container create 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , version=8.5, com.redhat.component=ubi8-container, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897) 2024-04-10T03:08:39.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:39 smithi152 ceph-mon[33523]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 635 op/s 2024-04-10T03:08:39.680 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 podman[77178]: 2024-04-10 03:08:39.402852536 +0000 UTC m=+0.408834821 container init 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.component=ubi8-container, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:08:39.680 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 podman[77178]: 2024-04-10 03:08:39.405966768 +0000 UTC m=+0.411949047 container start 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, version=8.5, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, distribution-scope=public, name=ubi8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., release=239.1651231664, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:08:39.680 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 bash[77178]: 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 2024-04-10T03:08:39.680 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:39.958 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:39 smithi102 ceph-mon[29301]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 635 op/s 2024-04-10T03:08:39.958 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:39 smithi102 ceph-mon[39651]: pgmap v9: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 635 op/s 2024-04-10T03:08:40.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[66856]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:08:39] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-10T03:08:40.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[77190]: logger=settings t=2024-04-10T03:08:39.69879635Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:08:40.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:39 smithi152 podman[77205]: 2024-04-10 03:08:39.747450587 +0000 UTC m=+0.032506632 container died 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-type=git, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, distribution-scope=public, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., release=239.1651231664, version=8.5) 2024-04-10T03:08:40.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:40 smithi152 podman[77205]: 2024-04-10 03:08:40.384048851 +0000 UTC m=+0.669104885 container remove 5e40471d4c8c77b9915dfda72884a7d1cd0b080280f48227f6e23d051ef64184 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , name=ubi8, description=Ceph Grafana Container, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, architecture=x86_64, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-10T03:08:40.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:40 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:08:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3057831092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3057831092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:40 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:40.847 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:40 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3057831092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:40.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:40 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:40 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:40.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:40 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:40.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:40 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:08:41.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:40 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:41.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:40 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.078s CPU time. 2024-04-10T03:08:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:41 smithi152 ceph-mon[33523]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:41 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:08:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:41 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:08:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:41 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[29301]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[39651]: pgmap v10: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:08:42.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:41 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:42 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:43.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:42 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 1.05k op/s 2024-04-10T03:08:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:08:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:43 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2342710546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:44.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:44.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 1.05k op/s 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2342710546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: pgmap v11: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 580 op/s 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: pgmap v12: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 1.05k op/s 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:08:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:43 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2342710546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:44.602 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:44.380Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-10T03:08:44.602 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:44.381Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-10T03:08:44.602 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:44.381Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-10T03:08:44.602 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:44.385Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-10T03:08:44.602 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:44.385Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-04-10T03:08:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:44 smithi152 ceph-mon[33523]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:08:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:44 smithi102 ceph-mon[29301]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:08:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:44 smithi102 ceph-mon[39651]: Reconfiguring daemon prometheus.a on smithi152 2024-04-10T03:08:45.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:45 smithi152 ceph-mon[33523]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:45.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:45 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1422742985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:45 smithi102 ceph-mon[39651]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:45 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1422742985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:45 smithi102 ceph-mon[29301]: pgmap v13: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:45 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1422742985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:47.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:47 smithi152 ceph-mon[33523]: pgmap v14: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:48.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:47 smithi102 ceph-mon[39651]: pgmap v14: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:48.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:47 smithi102 ceph-mon[29301]: pgmap v14: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-10T03:08:48.511 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 systemd[1]: Stopping Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:48.995 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3922528534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:48.996 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:08:48.996 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-10T03:08:48.996 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.681Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.682Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.682Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-10T03:08:48.997 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.682Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-10T03:08:48.998 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[74896]: ts=2024-04-10T03:08:48.682Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-10T03:08:48.998 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:48 smithi152 podman[78276]: 2024-04-10 03:08:48.691221971 +0000 UTC m=+0.149850621 container died 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:48 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3922528534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:48 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3922528534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:49.261 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 podman[78276]: 2024-04-10 03:08:49.131623126 +0000 UTC m=+0.590251773 container cleanup 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:49.262 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 bash[78276]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a 2024-04-10T03:08:49.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 podman[78287]: 2024-04-10 03:08:49.365338487 +0000 UTC m=+0.672863656 container remove 3b4d3921091373ffacfdb4bb0bb8e8d60f9412f9d891bf8f277298a36415b7a2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:49 smithi152 ceph-mon[33523]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:50.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-10T03:08:50.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 systemd[1]: Stopped Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:50.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service: Consumed 1.612s CPU time. 2024-04-10T03:08:50.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:49 smithi152 systemd[1]: Starting Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:49 smithi102 ceph-mon[39651]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:49 smithi102 ceph-mon[29301]: pgmap v15: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:50.453 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 podman[78360]: 2024-04-10 03:08:50.058234506 +0000 UTC m=+0.051037384 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-10T03:08:50.453 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 podman[78360]: 2024-04-10 03:08:50.183363082 +0000 UTC m=+0.176165961 container create a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:50.453 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 podman[78360]: 2024-04-10 03:08:50.455947342 +0000 UTC m=+0.448750225 container init a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 podman[78360]: 2024-04-10 03:08:50.466784975 +0000 UTC m=+0.459587856 container start a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.518Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.518Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.518Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi152 (none))" 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.518Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-10T03:08:50.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.518Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.521Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.521Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.526Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.526Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.528Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.529Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.315µs 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.529Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.529Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.572Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 bash[78360]: a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.576Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.577Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.577Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.383µs wal_replay_duration=48.354237ms wbl_replay_duration=158ns total_replay_duration=48.412313ms 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 systemd[1]: Started Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.581Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-10T03:08:50.764 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.581Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-10T03:08:50.765 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.581Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-10T03:08:50.765 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.624Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=42.330462ms db_storage=1.11µs remote_storage=1.64µs web_handler=789ns query_engine=849ns scrape=570.013µs scrape_sd=124.91µs notify=15.526µs notify_sd=13.761µs rules=41.225997ms tracing=8.174µs 2024-04-10T03:08:50.765 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.625Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-10T03:08:50.765 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:08:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:08:50.625Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-10T03:08:51.385 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-10T03:08:51.386 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:51.386 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.078s CPU time. 2024-04-10T03:08:51.386 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:51.386 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 podman[78492]: 2024-04-10 03:08:51.289476384 +0000 UTC m=+0.047303143 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:08:51.639 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 podman[78492]: 2024-04-10 03:08:51.421711019 +0000 UTC m=+0.179537774 container create de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.expose-services=, com.redhat.component=ubi8-container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.tags=base rhel8) 2024-04-10T03:08:51.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:51.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:51.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2823052500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:51.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:51.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:51.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:51.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:51 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:51.982 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 podman[78492]: 2024-04-10 03:08:51.708731023 +0000 UTC m=+0.466557780 container init de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, distribution-scope=public, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897) 2024-04-10T03:08:51.982 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 podman[78492]: 2024-04-10 03:08:51.712900051 +0000 UTC m=+0.470726802 container start de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.buildah.version=1.32.0, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, architecture=x86_64, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, version=8.5, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-10T03:08:51.982 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 bash[78492]: de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 2024-04-10T03:08:52.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2823052500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: pgmap v16: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2823052500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-10T03:08:52.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:51 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:08:52.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:51 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:52.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:52 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[78514]: logger=settings t=2024-04-10T03:08:52.000510014Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:08:52.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:52 smithi152 podman[78552]: 2024-04-10 03:08:52.050324792 +0000 UTC m=+0.039660558 container died de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, distribution-scope=public, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.tags=base rhel8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-04-29T11:48:11.055897, name=ubi8, vendor=Red Hat, Inc., release=239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:08:52.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:52 smithi152 podman[78552]: 2024-04-10 03:08:52.690238119 +0000 UTC m=+0.679573881 container remove de819ee7fe058a1aeefb8ba6f1f7f0e139df20e7a33b947cefcee3d7beca1589 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.openshift.expose-services=, vendor=Red Hat, Inc., release=239.1651231664, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-10T03:08:53.012 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:52 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:08:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:53 smithi152 ceph-mon[33523]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-10T03:08:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:53 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/753628178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:53 smithi102 ceph-mon[39651]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-10T03:08:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:53 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/753628178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:53 smithi102 ceph-mon[29301]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-10T03:08:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:53 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/753628178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:54.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:53 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:08:54.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:08:53 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.093s CPU time. 2024-04-10T03:08:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:54 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:54 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:54 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:55.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:54 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:08:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:08:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:54 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:08:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:55 smithi152 ceph-mon[33523]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:55 smithi152 ceph-mon[33523]: Upgrade: Updating mgr.x 2024-04-10T03:08:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:55 smithi152 ceph-mon[33523]: Deploying daemon mgr.x on smithi152 2024-04-10T03:08:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[39651]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[39651]: Upgrade: Updating mgr.x 2024-04-10T03:08:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[39651]: Deploying daemon mgr.x on smithi152 2024-04-10T03:08:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[29301]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[29301]: Upgrade: Updating mgr.x 2024-04-10T03:08:56.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:55 smithi102 ceph-mon[29301]: Deploying daemon mgr.x on smithi152 2024-04-10T03:08:57.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:56 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:57.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:56 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:57.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:56 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3540685784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:57.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:56 smithi152 ceph-mon[33523]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:57.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:56 smithi152 systemd[1]: Stopping Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:57.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:57.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:57.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3540685784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:57.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[39651]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:57.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:08:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:08:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/3540685784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:56 smithi102 ceph-mon[29301]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-10T03:08:57.761 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:57 smithi152 podman[78768]: 2024-04-10 03:08:57.285274751 +0000 UTC m=+0.195897502 container died da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:08:58.185 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:57 smithi152 podman[78768]: 2024-04-10 03:08:57.885574621 +0000 UTC m=+0.796197371 container cleanup da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-10T03:08:58.185 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:57 smithi152 bash[78768]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x 2024-04-10T03:08:58.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 podman[78779]: 2024-04-10 03:08:58.187669396 +0000 UTC m=+0.902067606 container remove da36e275413d0ec4d5c61e4ae6319c97e92216c32e7d572050cd16b8f17b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-10T03:08:58.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:08:58.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-10T03:08:58.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 systemd[1]: Stopped Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:58.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service: Consumed 43.306s CPU time. 2024-04-10T03:08:59.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:58 smithi152 systemd[1]: Starting Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:08:59.392 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 podman[78894]: 2024-04-10 03:08:58.984390584 +0000 UTC m=+0.035950348 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:08:59.392 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 podman[78894]: 2024-04-10 03:08:59.114625163 +0000 UTC m=+0.166184882 container create 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403) 2024-04-10T03:08:59.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:59 smithi152 ceph-mon[33523]: pgmap v20: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:59.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:08:59 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2067407898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:59.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 podman[78894]: 2024-04-10 03:08:59.395434437 +0000 UTC m=+0.446994161 container init 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-10T03:08:59.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 podman[78894]: 2024-04-10 03:08:59.406138322 +0000 UTC m=+0.457698042 container start 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:08:59.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 bash[78894]: 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc 2024-04-10T03:08:59.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:08:59 smithi152 systemd[1]: Started Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:08:59.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:59 smithi102 ceph-mon[39651]: pgmap v20: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:59.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:08:59 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2067407898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:08:59.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:59 smithi102 ceph-mon[29301]: pgmap v20: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:08:59.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:08:59 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2067407898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:00.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:00 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:00.352+0000 7f4ec3dc1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:09:00.940 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.940 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.940 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:00.941 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.941 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.941 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:00.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:00 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:00 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:00.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:00 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:00.962 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:00 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:00.518+0000 7f4ec3dc1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:09:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:09:00] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:09:01.890 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:01 smithi102 ceph-mon[29301]: pgmap v21: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:09:01.890 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:01 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/525454743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:01.890 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:01 smithi102 ceph-mon[39651]: pgmap v21: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:09:01.890 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:01 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/525454743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:01 smithi152 ceph-mon[33523]: pgmap v21: 89 pgs: 89 active+clean; 971 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-10T03:09:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:01 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/525454743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:02.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:01 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:01.641+0000 7f4ec3dc1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:09:02.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:01 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:01.801+0000 7f4ec3dc1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:09:02.345 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:02 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:02.043+0000 7f4ec3dc1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:09:02.346 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:02 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:02.207+0000 7f4ec3dc1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:09:02.346 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:02 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:02.347+0000 7f4ec3dc1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:09:03.137 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:02 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:02.807+0000 7f4ec3dc1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:09:03.137 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:02.998+0000 7f4ec3dc1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:03.389 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:03.142+0000 7f4ec3dc1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:09:03.390 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:03.305+0000 7f4ec3dc1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:09:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:03 smithi152 ceph-mon[33523]: pgmap v22: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-10T03:09:03.762 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:03 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:03.580+0000 7f4ec3dc1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:09:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:03 smithi102 ceph-mon[29301]: pgmap v22: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-10T03:09:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:03 smithi102 ceph-mon[39651]: pgmap v22: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-10T03:09:04.511 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-10T03:09:04.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:04.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.093s CPU time. 2024-04-10T03:09:04.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:04.798 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:04 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4066196484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:04.798 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 podman[79406]: 2024-04-10 03:09:04.666774372 +0000 UTC m=+0.370907586 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:09:04.798 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:04 smithi152 podman[79406]: 2024-04-10 03:09:04.800952986 +0000 UTC m=+0.505086177 container create b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.expose-services=, architecture=x86_64, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, version=8.5, release=239.1651231664) 2024-04-10T03:09:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:04 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4066196484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:04 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/4066196484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:05.514 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:05.226+0000 7f4ec3dc1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:09:05.514 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:05 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:05.439+0000 7f4ec3dc1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:09:05.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:05 smithi102 ceph-mon[29301]: pgmap v23: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:05 smithi102 ceph-mon[39651]: pgmap v23: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:05.906 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:05 smithi152 ceph-mon[33523]: pgmap v23: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:05.906 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:05 smithi152 podman[79406]: 2024-04-10 03:09:05.808394819 +0000 UTC m=+1.512528019 container init b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, version=8.5, description=Ceph Grafana Container, distribution-scope=public, architecture=x86_64, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=) 2024-04-10T03:09:05.906 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:05 smithi152 podman[79406]: 2024-04-10 03:09:05.814340859 +0000 UTC m=+1.518474051 container start b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, distribution-scope=public, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.32.0, maintainer=Paul Cuzner , name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5) 2024-04-10T03:09:06.182 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:06.006+0000 7f4ec3dc1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:09:06.182 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:05 smithi152 bash[79406]: b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 2024-04-10T03:09:06.182 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:05 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:06.449 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:06.184+0000 7f4ec3dc1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:09:06.449 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[79469]: logger=settings t=2024-04-10T03:09:06.253846289Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:09:06.449 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:06 smithi152 podman[79552]: 2024-04-10 03:09:06.331837542 +0000 UTC m=+0.054444340 container died b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, name=ubi8, vendor=Red Hat, Inc., vcs-type=git, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-04-10T03:09:06.761 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:06.548+0000 7f4ec3dc1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[29301]: pgmap v24: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/546335677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:07.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[39651]: pgmap v24: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:07.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:06 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/546335677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-mon[33523]: pgmap v24: 89 pgs: 89 active+clean; 730 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2024-04-10T03:09:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/546335677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:07.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:06 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:06.876+0000 7f4ec3dc1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:09:07.379 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:07 smithi152 podman[79552]: 2024-04-10 03:09:07.076028295 +0000 UTC m=+0.798635078 container remove b48efbfdd4f38c1537bbed0d04b010b10d9283df65d7a31cd342ea59493916b0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897) 2024-04-10T03:09:07.379 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:09:07.761 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:07.381+0000 7f4ec3dc1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:09:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.204s CPU time. 2024-04-10T03:09:08.247 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:07 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:07.825+0000 7f4ec3dc1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:08.247 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:08.248+0000 7f4ec3dc1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:09:08.875 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:08 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:08.875 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:08 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:08.875 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:09:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:09:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:09:09.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:08 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:09.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:08 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:09.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:08 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:09.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:08 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:10.000 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:09 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:09.628+0000 7f4ec3dc1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:09:10.001 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:09 smithi152 ceph-mon[33523]: pgmap v25: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-04-10T03:09:10.001 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:09 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1667811385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:09 smithi102 ceph-mon[39651]: pgmap v25: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-04-10T03:09:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:09 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1667811385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:09 smithi102 ceph-mon[29301]: pgmap v25: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-04-10T03:09:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:09 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1667811385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:11.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:10 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:10.566+0000 7f4ec3dc1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:09:11.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:10 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:10.746+0000 7f4ec3dc1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:09:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:09:10] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:11.146+0000 7f4ec3dc1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:11] ENGINE Bus STARTING 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: CherryPy Checker: 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: The Application mounted at '' has an empty config. 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:11] ENGINE Serving on http://:::9283 2024-04-10T03:09:11.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:11] ENGINE Bus STARTED 2024-04-10T03:09:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: pgmap v26: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: Standby manager daemon x restarted 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:11 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: pgmap v26: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2024-04-10T03:09:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: Standby manager daemon x restarted 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: Standby manager daemon x started 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[29301]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: pgmap v26: 89 pgs: 89 active+clean; 526 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: Standby manager daemon x restarted 2024-04-10T03:09:12.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T03:09:12.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:12.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:12.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:12.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:11 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/1190594988' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:12.997 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:12 smithi152 ceph-mon[33523]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-10T03:09:12.997 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:12 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/857292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:12 smithi102 ceph-mon[29301]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-10T03:09:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:12 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/857292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:12 smithi102 ceph-mon[39651]: mgrmap e34: y(active, since 45s), standbys: x 2024-04-10T03:09:13.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:12 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/857292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:14.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: pgmap v27: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-10T03:09:14.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:09:14.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:13 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-10T03:09:14.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: pgmap v27: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: pgmap v27: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:09:14.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:13 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-10T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[29301]: pgmap v28: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[29301]: pgmap v29: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[29301]: Upgrade: Setting container_image for all mgr 2024-04-10T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[29301]: Upgrade: It appears safe to stop mon.a 2024-04-10T03:09:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/2701666180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[39651]: pgmap v28: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[39651]: pgmap v29: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[39651]: Upgrade: Setting container_image for all mgr 2024-04-10T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[39651]: Upgrade: It appears safe to stop mon.a 2024-04-10T03:09:14.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:14 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2701666180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:14 smithi152 ceph-mon[33523]: pgmap v28: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-04-10T03:09:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:14 smithi152 ceph-mon[33523]: pgmap v29: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:14 smithi152 ceph-mon[33523]: Upgrade: Setting container_image for all mgr 2024-04-10T03:09:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:14 smithi152 ceph-mon[33523]: Upgrade: It appears safe to stop mon.a 2024-04-10T03:09:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:14 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2701666180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[39651]: pgmap v30: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[39651]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[29301]: pgmap v30: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[29301]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:16.921 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:09:16.922 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:09:16.922 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:16 smithi102 ceph-mon[29301]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:17.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:16 smithi152 ceph-mon[33523]: pgmap v30: 89 pgs: 89 active+clean; 302 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-04-10T03:09:17.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:16 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='mgr.y' 2024-04-10T03:09:17.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:16 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:09:17.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:16 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:09:17.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:16 smithi152 ceph-mon[33523]: from='mgr.15165 172.21.15.102:0/1541867047' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:17.583 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Removed 127198 objects 2024-04-10T03:09:17.583 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Clean up completed and total clean up time :158.602 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[39651]: Upgrade: Updating mon.a 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[39651]: Deploying daemon mon.a on smithi102 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1030083309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[29301]: Upgrade: Updating mon.a 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[29301]: Deploying daemon mon.a on smithi102 2024-04-10T03:09:17.969 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:17 smithi102 ceph-mon[29301]: from='client.? 172.21.15.102:0/1030083309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:17.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:17 smithi152 ceph-mon[33523]: Upgrade: Updating mon.a 2024-04-10T03:09:17.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:17 smithi152 ceph-mon[33523]: Deploying daemon mon.a on smithi102 2024-04-10T03:09:17.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:17 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1030083309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:17.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:17 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-10T03:09:17.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:17 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:17.988 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:17 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.204s CPU time. 2024-04-10T03:09:17.989 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:17 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:18.250 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:17 smithi152 podman[82381]: 2024-04-10 03:09:17.893284393 +0000 UTC m=+0.017085631 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:09:18.250 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 podman[82381]: 2024-04-10 03:09:18.017300804 +0000 UTC m=+0.141102027 container create f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.expose-services=, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc.) 2024-04-10T03:09:18.250 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 podman[82381]: 2024-04-10 03:09:18.253101834 +0000 UTC m=+0.376903057 container init f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-10T03:09:18.511 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 podman[82381]: 2024-04-10 03:09:18.255756611 +0000 UTC m=+0.379557843 container start f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, name=ubi8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-10T03:09:18.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 bash[82381]: f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 2024-04-10T03:09:18.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:18.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[82396]: logger=settings t=2024-04-10T03:09:18.410889403Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:09:18.512 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:18 smithi152 podman[82413]: 2024-04-10 03:09:18.436328322 +0000 UTC m=+0.018394976 container died f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, architecture=x86_64, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git) 2024-04-10T03:09:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:18 smithi102 systemd[1]: Stopping Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:18.976 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[29297]: 2024-04-10T03:09:18.644+0000 7f43f7ec8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:09:18.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[29297]: 2024-04-10T03:09:18.644+0000 7f43f7ec8700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-10T03:09:18.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:18 smithi102 podman[94174]: 2024-04-10 03:09:18.665436417 +0000 UTC m=+0.115307582 container died 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-10T03:09:19.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:09:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:09:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:09:19.061 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-04-10T03:09:19.061 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-04-10T03:09:19.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:19 smithi152 podman[82413]: 2024-04-10 03:09:19.341571292 +0000 UTC m=+0.923637938 container remove f4a0062b619407201ffe16f3c026c83ab953d5e62d39e87860e5c99651f5c945 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, com.redhat.component=ubi8-container, description=Ceph Grafana Container, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.tags=base rhel8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-10T03:09:19.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:19 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:09:19.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:19 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:09:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:19 smithi102 podman[94174]: 2024-04-10 03:09:19.488134081 +0000 UTC m=+0.938005240 container cleanup 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-10T03:09:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:19 smithi102 bash[94174]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a 2024-04-10T03:09:19.954 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:20.188 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:19 smithi102 podman[94188]: 2024-04-10 03:09:19.881545013 +0000 UTC m=+1.218343539 container remove 2b567b259d3dff550a0ef10b9eeec21a1dd0c2f30283d65f66137987ce1882c5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0) 2024-04-10T03:09:21.191 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:09:20] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:09:21.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:21 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-10T03:09:21.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:21 smithi102 systemd[1]: Stopped Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:21.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:21 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service: Consumed 13.831s CPU time. 2024-04-10T03:09:22.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:21 smithi102 systemd[1]: Starting Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:22.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 podman[94512]: 2024-04-10 03:09:22.100741258 +0000 UTC m=+0.019190412 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:09:22.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 podman[94512]: 2024-04-10 03:09:22.210691138 +0000 UTC m=+0.129140316 container create c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux ) 2024-04-10T03:09:22.578 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 podman[94512]: 2024-04-10 03:09:22.430056954 +0000 UTC m=+0.348506091 container init c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240403) 2024-04-10T03:09:22.578 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 podman[94512]: 2024-04-10 03:09:22.433728325 +0000 UTC m=+0.352177464 container start c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: set uid:gid to 167:167 (ceph:ceph) 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev), process ceph-mon, pid 2 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: pidfile_write: ignore empty --pid-file 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: load: jerasure load: lrc 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: RocksDB version: 7.9.2 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Git sha 0 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Compile date 2024-04-09 01:35:31 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: DB SUMMARY 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: DB Session ID: 89XOQ64YI3OJD5LFYX9X 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: CURRENT file: CURRENT 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: IDENTITY file: IDENTITY 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1168 Bytes 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-04-10T03:09:22.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 795596 ; 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.error_if_exists: 0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.create_if_missing: 0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.paranoid_checks: 1 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.env: 0x5578f8501c60 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.fs: PosixFileSystem 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.info_log: 0x5578f94f3ae0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_file_opening_threads: 16 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.statistics: (nil) 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.use_fsync: 0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_log_file_size: 0 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-10T03:09:22.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.keep_log_file_num: 1000 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.recycle_log_file_num: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_fallocate: 1 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_mmap_reads: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_mmap_writes: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.use_direct_reads: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.create_missing_column_families: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.db_log_dir: 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.wal_dir: 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-10T03:09:22.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.advise_random_on_open: 1 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.db_write_buffer_size: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.write_buffer_manager: 0x5578f94c6820 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.rate_limiter: (nil) 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.wal_recovery_mode: 2 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enable_thread_tracking: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enable_pipelined_write: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.unordered_write: 0 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-10T03:09:22.582 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-10T03:09:22.583 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-10T03:09:22.583 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.row_cache: None 2024-04-10T03:09:22.583 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.wal_filter: None 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_ingest_behind: 0 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.two_write_queues: 0 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.manual_wal_flush: 0 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.wal_compression: 0 2024-04-10T03:09:22.584 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.atomic_flush: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.log_readahead_size: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.best_efforts_recovery: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.allow_data_in_errors: 0 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.db_host_id: __hostname__ 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_background_jobs: 2 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_background_compactions: -1 2024-04-10T03:09:22.585 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_subcompactions: 1 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_total_wal_size: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_open_files: -1 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bytes_per_sync: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_readahead_size: 0 2024-04-10T03:09:22.586 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_background_flushes: -1 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Compression algorithms supported: 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kZSTD supported: 0 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kXpressCompression supported: 0 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kLZ4HCCompression supported: 1 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kZlibCompression supported: 1 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kSnappyCompression supported: 1 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kLZ4Compression supported: 1 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: kBZip2Compression supported: 0 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-10T03:09:22.587 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.merge_operator: 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_filter: None 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_filter_factory: None 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.sst_partitioner_factory: None 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5578f94f3be0) 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_type: 0 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: data_block_index_type: 0 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_shortening: 1 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-10T03:09:22.588 INFO:journalctl@ceph.mon.a.smithi102.stdout: checksum: 4 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: no_block_cache: 0 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache: 0x5578f94f7090 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_options: 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: capacity : 536870912 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: num_shard_bits : 4 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: strict_capacity_limit : 0 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_cache_compressed: (nil) 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: persistent_cache: (nil) 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_size: 4096 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_size_deviation: 10 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_restart_interval: 16 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: index_block_restart_interval: 1 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: metadata_block_size: 4096 2024-04-10T03:09:22.589 INFO:journalctl@ceph.mon.a.smithi102.stdout: partition_filters: 0 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: use_delta_encoding: 1 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: filter_policy: bloomfilter 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: whole_key_filtering: 1 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: verify_compression: 0 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: format_version: 5 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: enable_index_compression: 1 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: block_align: 0 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: max_auto_readahead_size: 262144 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: prepopulate_block_cache: 0 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: initial_auto_readahead_size: 8192 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout: num_file_reads_for_auto_readahead: 2 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.write_buffer_size: 33554432 2024-04-10T03:09:22.590 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_write_buffer_number: 2 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression: NoCompression 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression: Disabled 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.prefix_extractor: nullptr 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.num_levels: 7 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-10T03:09:22.591 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.level: 32767 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.strategy: 0 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.enabled: false 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-10T03:09:22.592 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.target_file_size_base: 67108864 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-10T03:09:22.593 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.arena_block_size: 1048576 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.disable_auto_compactions: 0 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.table_properties_collectors: 2024-04-10T03:09:22.594 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.inplace_update_support: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.bloom_locality: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.max_successive_merges: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.paranoid_file_checks: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.force_consistency_checks: 1 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.report_bg_io_stats: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.ttl: 2592000 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enable_blob_files: false 2024-04-10T03:09:22.595 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.min_blob_size: 0 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_file_size: 268435456 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.blob_file_starting_level: 0 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12125, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a6469ba1-df75-42dc-92ce-cb784a6c871b 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718562464060, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-04-10T03:09:22.596 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718562469089, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 739514, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12119, "largest_seqno": 12670, "table_properties": {"data_size": 736216, "index_size": 1580, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 6291, "raw_average_key_size": 23, "raw_value_size": 730551, "raw_average_value_size": 2767, "num_data_blocks": 72, "num_entries": 264, "num_filter_entries": 264, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712718562, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a6469ba1-df75-42dc-92ce-cb784a6c871b", "db_session_id": "89XOQ64YI3OJD5LFYX9X", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718562469180, "job": 1, "event": "recovery_finished"} 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5578f95e8000 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: DB pointer 0x5578f95d2000 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** DB Stats ** 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:09:22.597 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: L0 1/0 722.18 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 241.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: L6 1/0 7.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Sum 2/0 7.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 241.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 241.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 241.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-10T03:09:22.598 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Cumulative compaction: 0.00 GB write, 92.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Interval compaction: 0.00 GB write, 92.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Block cache BinnedLRUCache@0x5578f94f7090#2 capacity: 512.00 MB usage: 2.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-04-10T03:09:22.599 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: starting mon.a rank 0 at public addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] at bind addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???) e3 preinit fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).mds e9 new map 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).mds e9 print_map 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: e9 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: legacy client fscid: 1 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: Filesystem 'foo' (1) 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: fs_name foo 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: epoch 8 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: created 2024-04-10T03:03:52.164573+0000 2024-04-10T03:09:22.600 INFO:journalctl@ceph.mon.a.smithi102.stdout: modified 2024-04-10T03:04:03.738495+0000 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: tableserver 0 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: root 0 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: session_timeout 60 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: session_autoclose 300 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: max_file_size 1099511627776 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: max_xattr_size 65536 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: required_client_features {} 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: last_failure 0 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: last_failure_osd_epoch 0 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: max_mds 1 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: in 0 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: up {0=24545} 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: failed 2024-04-10T03:09:22.601 INFO:journalctl@ceph.mon.a.smithi102.stdout: damaged 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: stopped 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: data_pools [4] 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: metadata_pool 3 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: inline_data disabled 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: balancer 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: bal_rank_mask -1 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: standby_count_wanted 1 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: qdb_cluster leader: 0 members: 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: [mds.foo.smithi102.exkvif{0:24545} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: Standby daemons: 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout: [mds.foo.smithi152.adssru{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:09:22.602 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 ceph-mon[94535]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 bash[94512]: c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 2024-04-10T03:09:22.603 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:22 smithi102 systemd[1]: Started Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:25.564 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_0' removed 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ignoring --setuser ceph since I am not root 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ignoring --setgroup ceph since I am not root 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mgr[84858]: -- 172.21.15.102:0/2749771274 <== mon.2 v2:172.21.15.102:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55f8ba8c81a0 con 0x55f8ba89ec00 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mgr[84858]: -- 172.21.15.102:0/2749771274 <== mon.2 v2:172.21.15.102:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f8ba8ca5a0 con 0x55f8ba89ec00 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: pgmap v32: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/310335884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: pgmap v33: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/4103737571' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/262647225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: pgmap v34: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3211040101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: mon.a calling monitor election 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='client.? ' entity='' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: monmap epoch 3 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: min_mon_release 17 (quincy) 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: election_strategy: 1 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:09:25.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: mgrmap e34: y(active, since 59s), standbys: x 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: daemon grafana.a on smithi152 is in unknown state 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: mgrmap e35: y(active, since 59s), standbys: x 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[39651]: from='mgr.15165 ' entity='' 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: pgmap v32: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/310335884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: pgmap v33: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:25.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4103737571' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/262647225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: pgmap v34: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3211040101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: mon.a calling monitor election 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='client.? ' entity='' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: monmap epoch 3 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: min_mon_release 17 (quincy) 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: election_strategy: 1 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:09:25.795 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: mgrmap e34: y(active, since 59s), standbys: x 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: daemon grafana.a on smithi152 is in unknown state 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: mgrmap e35: y(active, since 59s), standbys: x 2024-04-10T03:09:25.796 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:25 smithi102 ceph-mon[94535]: from='mgr.15165 ' entity='' 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: ignoring --setuser ceph since I am not root 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: ignoring --setgroup ceph since I am not root 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mgr[78909]: -- 172.21.15.152:0/498240462 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d5c09845a0 con 0x55d5c0958800 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: pgmap v32: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/310335884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: pgmap v33: 89 pgs: 89 active+clean; 39 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4103737571' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/262647225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: pgmap v34: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3211040101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: mon.a calling monitor election 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='client.? ' entity='' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: monmap epoch 3 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: min_mon_release 17 (quincy) 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: election_strategy: 1 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:09:26.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: osdmap e76: 8 total, 8 up, 8 in 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: mgrmap e34: y(active, since 59s), standbys: x 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: daemon grafana.a on smithi152 is in unknown state 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: mgrmap e35: y(active, since 59s), standbys: x 2024-04-10T03:09:26.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:25 smithi152 ceph-mon[33523]: from='mgr.15165 ' entity='' 2024-04-10T03:09:26.193 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:26.100+0000 7f1bb9dc6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:09:26.291 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-10T03:09:26.292 INFO:tasks.radosbench:Beginning radosbench... 2024-04-10T03:09:26.292 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-04-10T03:09:26.293 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool create unique_pool_1 16 2024-04-10T03:09:26.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:26.104+0000 7f0685f19200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-10T03:09:26.512 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:26.198+0000 7f0685f19200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:09:26.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:26.195+0000 7f1bb9dc6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-10T03:09:26.582 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:26.859 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-mon[39651]: from='client.? ' entity='' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:09:26.859 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-mon[39651]: osdmap e77: 8 total, 8 up, 8 in 2024-04-10T03:09:26.859 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-mon[94535]: from='client.? ' entity='' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:09:26.859 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-mon[94535]: osdmap e77: 8 total, 8 up, 8 in 2024-04-10T03:09:26.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-mon[33523]: from='client.? ' entity='' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:09:26.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-mon[33523]: osdmap e77: 8 total, 8 up, 8 in 2024-04-10T03:09:26.964 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 1 2024-04-10T03:09:27.184 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:26 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:26.942+0000 7f1bb9dc6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:09:27.184 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.044+0000 7f1bb9dc6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:09:27.184 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.186+0000 7f1bb9dc6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:09:27.221 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:26.885+0000 7f0685f19200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-10T03:09:27.221 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:26.985+0000 7f0685f19200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-10T03:09:27.221 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.127+0000 7f0685f19200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-10T03:09:27.221 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.223+0000 7f0685f19200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:09:27.490 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.283+0000 7f1bb9dc6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-10T03:09:27.490 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.367+0000 7f1bb9dc6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:09:27.511 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.306+0000 7f0685f19200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-10T03:09:27.741 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1568426707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:27.742 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1568426707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:27.742 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.633+0000 7f1bb9dc6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:09:27.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1568426707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:09:27.867 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.575+0000 7f0685f19200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-10T03:09:27.867 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.685+0000 7f0685f19200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:27.867 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.772+0000 7f0685f19200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:09:27.867 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:27.868+0000 7f0685f19200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:09:28.019 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.743+0000 7f1bb9dc6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:28.019 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.832+0000 7f1bb9dc6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-10T03:09:28.019 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:27 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:27.933+0000 7f1bb9dc6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-10T03:09:28.261 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:28.048+0000 7f0685f19200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:09:28.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:28 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:28.111+0000 7f1bb9dc6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-10T03:09:28.334 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:09:28.889 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:29.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:09:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:09:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:09:29.273 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:29.134+0000 7f1bb9dc6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:09:29.348 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:29.025+0000 7f0685f19200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-10T03:09:29.348 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:29.156+0000 7f0685f19200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:09:29.350 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_1' created 2024-04-10T03:09:29.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-10T03:09:29.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-10T03:09:29.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:29.275+0000 7f1bb9dc6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-10T03:09:29.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-10T03:09:29.599 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:29.506+0000 7f0685f19200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:09:29.912 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:29.630+0000 7f1bb9dc6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-10T03:09:29.912 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:29.724+0000 7f1bb9dc6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:09:29.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:29 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-10T03:09:29.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:29 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:29.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:29 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:29.999 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:29.601+0000 7f0685f19200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-10T03:09:29.999 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:29.811+0000 7f0685f19200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:30.191 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:29 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:29.928+0000 7f1bb9dc6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:30.192 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:30.122+0000 7f1bb9dc6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:09:30.254 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:30.021+0000 7f0685f19200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-10T03:09:30.254 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 podman[82573]: 2024-04-10 03:09:29.903762191 +0000 UTC m=+0.018128782 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:09:30.254 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 podman[82573]: 2024-04-10 03:09:30.042549435 +0000 UTC m=+0.156916014 container create e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, version=8.5, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.32.0, name=ubi8, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, release=239.1651231664, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-10T03:09:30.485 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:30.433+0000 7f1bb9dc6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:30.485 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-10T03:09:30.485 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[94535]: osdmap e78: 8 total, 8 up, 8 in 2024-04-10T03:09:30.485 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[94535]: osdmap e79: 8 total, 8 up, 8 in 2024-04-10T03:09:30.486 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[39651]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-10T03:09:30.486 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[39651]: osdmap e78: 8 total, 8 up, 8 in 2024-04-10T03:09:30.486 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-mon[39651]: osdmap e79: 8 total, 8 up, 8 in 2024-04-10T03:09:30.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-mon[33523]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-10T03:09:30.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-mon[33523]: osdmap e78: 8 total, 8 up, 8 in 2024-04-10T03:09:30.602 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-mon[33523]: osdmap e79: 8 total, 8 up, 8 in 2024-04-10T03:09:30.602 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:30.324+0000 7f0685f19200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 podman[82573]: 2024-04-10 03:09:30.412315701 +0000 UTC m=+0.526682286 container init e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, description=Ceph Grafana Container, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 podman[82573]: 2024-04-10 03:09:30.415172578 +0000 UTC m=+0.529539157 container start e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, version=8.5, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, release=239.1651231664) 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 bash[82573]: e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[82592]: logger=settings t=2024-04-10T03:09:30.574453703Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:09:30.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:30 smithi152 podman[82611]: 2024-04-10 03:09:30.604630951 +0000 UTC m=+0.021342414 container died e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, vcs-type=git, distribution-scope=public) 2024-04-10T03:09:30.737 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:30.705+0000 7f1bb9dc6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:30.862 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:30.614+0000 7f0685f19200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-10T03:09:31.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:30.962+0000 7f1bb9dc6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:09:31.155 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:30.863+0000 7f0685f19200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-10T03:09:31.257 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-04-10T03:09:31.407 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:31 smithi152 podman[82611]: 2024-04-10 03:09:31.158087757 +0000 UTC m=+0.574799231 container remove e120dc02542f6d2593c81cbe024dc0577520a3e865894cbc3982ec79ae09df4a (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vcs-type=git, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, distribution-scope=public, version=8.5, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-10T03:09:31.407 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:31 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:09:31.514 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:31.718 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:31 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:09:31.978 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:31.823+0000 7f1bb9dc6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:09:32.011 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:31.720+0000 7f0685f19200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-10T03:09:32.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-mon[33523]: osdmap e80: 8 total, 8 up, 8 in 2024-04-10T03:09:32.631 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:32.302+0000 7f0685f19200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:09:32.632 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:32.407+0000 7f0685f19200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:09:32.632 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:32.632+0000 7f0685f19200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:09:32.740 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-mon[94535]: osdmap e80: 8 total, 8 up, 8 in 2024-04-10T03:09:32.741 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-mon[39651]: osdmap e80: 8 total, 8 up, 8 in 2024-04-10T03:09:32.741 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.412+0000 7f1bb9dc6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-10T03:09:32.741 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.516+0000 7f1bb9dc6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-10T03:09:32.949 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:32] ENGINE Bus STARTING 2024-04-10T03:09:32.949 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: CherryPy Checker: 2024-04-10T03:09:32.949 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: The Application mounted at '' has an empty config. 2024-04-10T03:09:32.949 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: 2024-04-10T03:09:32.949 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:32] ENGINE Serving on http://:::9283 2024-04-10T03:09:32.950 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:09:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x[78905]: [10/Apr/2024:03:09:32] ENGINE Bus STARTED 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.742+0000 7f1bb9dc6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:09:32] ENGINE Bus STARTING 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b17999700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b17999700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b17999700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b17999700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b17999700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b1d9a5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b1d9a5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b1d9a5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.008 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b1d9a5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:32.887+0000 7f1b1d9a5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: CherryPy Checker: 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: The Application mounted at '' has an empty config. 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:09:32] ENGINE Serving on http://:::9283 2024-04-10T03:09:33.009 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:09:32] ENGINE Bus STARTED 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: Standby manager daemon x restarted 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: Standby manager daemon x started 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: Active manager daemon y restarted 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: Activating manager daemon y 2024-04-10T03:09:33.576 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: osdmap e81: 8 total, 8 up, 8 in 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: mgrmap e36: y(active, starting, since 0.00634534s), standbys: x 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:09:33.577 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: Manager daemon y is now available 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.578 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: Standby manager daemon x restarted 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: Standby manager daemon x started 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: Active manager daemon y restarted 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: Activating manager daemon y 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: osdmap e81: 8 total, 8 up, 8 in 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: mgrmap e36: y(active, starting, since 0.00634534s), standbys: x 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:09:33.579 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: Manager daemon y is now available 2024-04-10T03:09:33.580 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:33.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:33.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.581 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: Standby manager daemon x restarted 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: Standby manager daemon x started 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.? 172.21.15.152:0/2810844708' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: Active manager daemon y restarted 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: Activating manager daemon y 2024-04-10T03:09:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: osdmap e81: 8 total, 8 up, 8 in 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: mgrmap e36: y(active, starting, since 0.00634534s), standbys: x 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.adssru"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi102.exkvif"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-10T03:09:33.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: Manager daemon y is now available 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:33 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (2m) 60s ago 6m 16.2M - 0.25.0 c8568f914cd2 65832d2b0608 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 unknown 28s ago 6m - - 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (5m) 60s ago 5m 14.4M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (5m) 28s ago 5m 15.2M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443,9283,8765 running (34s) 28s ago 10m 68.3M - 19.0.0-2051-g3a866975 06a486fd9bae 10f3bdd94f98 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:8443,9283,8765 running (82s) 60s ago 12m 479M - 19.0.0-2051-g3a866975 06a486fd9bae ba38a088e643 2024-04-10T03:09:33.773 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (12m) 60s ago 12m 51.7M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 2b567b259d3d 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 running (11m) 28s ago 11m 44.2M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 7ecfd799af6d 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 running (10m) 60s ago 10m 40.8M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 8f8b8b481e8d 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (116s) 60s ago 6m 7964k - 1.5.0 0da6a335fe13 773a048f8daf 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (89s) 28s ago 6m 17.0M - 1.5.0 0da6a335fe13 6010c1e76260 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (10m) 60s ago 10m 366M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 984949346b35 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (10m) 60s ago 10m 243M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 13648c191186 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (9m) 60s ago 9m 307M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad f10dfba3072f 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (9m) 60s ago 9m 165M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 6f7249bbf2b8 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (8m) 28s ago 8m 368M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (8m) 28s ago 8m 401M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (8m) 28s ago 8m 208M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (7m) 28s ago 7m 250M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:09:33.774 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (43s) 28s ago 7m 40.2M - 2.43.0 a07b618ecd1d a7900639fc5e 2024-04-10T03:09:33.999 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:09:33.761+0000 7f1b6ff70700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-10T03:09:34.353 INFO:teuthology.orchestra.run.smithi102.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-04-10T03:09:34.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:34 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:09:34.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:34 smithi102 ceph-mon[39651]: mgrmap e37: y(active, since 1.01239s), standbys: x 2024-04-10T03:09:34.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:09:34.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:34 smithi102 ceph-mon[94535]: mgrmap e37: y(active, since 1.01239s), standbys: x 2024-04-10T03:09:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:34 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:09:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:34 smithi152 ceph-mon[33523]: mgrmap e37: y(active, since 1.01239s), standbys: x 2024-04-10T03:09:35.417 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade pause' 2024-04-10T03:09:35.647 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: [10/Apr/2024:03:09:33] ENGINE Bus STARTING 2024-04-10T03:09:35.647 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: [10/Apr/2024:03:09:33] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:09:35.647 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: [10/Apr/2024:03:09:33] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: [10/Apr/2024:03:09:33] ENGINE Bus STARTED 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[94535]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: [10/Apr/2024:03:09:33] ENGINE Bus STARTING 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: [10/Apr/2024:03:09:33] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: [10/Apr/2024:03:09:33] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:09:35.648 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: [10/Apr/2024:03:09:33] ENGINE Bus STARTED 2024-04-10T03:09:35.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:09:35.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:09:35.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:09:35.649 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:35 smithi102 ceph-mon[39651]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:09:35.680 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: [10/Apr/2024:03:09:33] ENGINE Bus STARTING 2024-04-10T03:09:35.680 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: [10/Apr/2024:03:09:33] ENGINE Serving on https://172.21.15.102:7150 2024-04-10T03:09:35.680 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: [10/Apr/2024:03:09:33] ENGINE Serving on http://172.21.15.102:8765 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: [10/Apr/2024:03:09:33] ENGINE Bus STARTED 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1253080107' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:09:35.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:35 smithi152 ceph-mon[33523]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-10T03:09:35.921 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:36.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[94535]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:36.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[94535]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[94535]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[94535]: Cluster is now healthy 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[39651]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[39651]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[39651]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:09:36.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:36 smithi102 ceph-mon[39651]: Cluster is now healthy 2024-04-10T03:09:36.752 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:36 smithi152 ceph-mon[33523]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:36.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:36 smithi152 ceph-mon[33523]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-10T03:09:36.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:36 smithi152 ceph-mon[33523]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:09:36.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:36 smithi152 ceph-mon[33523]: Cluster is now healthy 2024-04-10T03:09:36.825 DEBUG:teuthology.orchestra.run.smithi102:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-04-10T03:09:36.828 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-10T03:09:36.828 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-10T03:09:36.872 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:hints = 1 2024-04-10T03:09:36.872 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-10T03:09:36.872 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object prefix: benchmark_data_smithi102_95822 2024-04-10T03:09:36.872 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:09:36.872 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 0 0 0 0 0 0 - 0 2024-04-10T03:09:37.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 1 16 3991 3975 248.31 248.438 0.00621675 0.00401523 2024-04-10T03:09:38.487 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[39651]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[39651]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[94535]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.488 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:38 smithi102 ceph-mon[94535]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-10T03:09:38.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:38 smithi152 ceph-mon[33523]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:38.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:38 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:38 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:38.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:38 smithi152 ceph-mon[33523]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-10T03:09:38.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 2 16 7323 7307 228.275 208.25 0.00473919 0.00437326 2024-04-10T03:09:39.275 INFO:teuthology.orchestra.run.smithi102.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:09:39.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 3 16 10542 10526 219.24 201.188 0.00585503 0.00455263 2024-04-10T03:09:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:40.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:09:40.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:39 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: pgmap v7: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:09:40.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:39 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:40.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 4 16 13742 13726 214.427 200 0.00391305 0.00465669 2024-04-10T03:09:41.183 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'sleep 60' 2024-04-10T03:09:41.249 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:09:40] "GET /metrics HTTP/1.1" 200 33499 "" "Prometheus/2.43.0" 2024-04-10T03:09:41.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:41 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.530 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:41 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.657 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:09:41.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:41 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:41 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:41 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-10T03:09:41.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:41 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:41.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:41 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:41.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:41 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:41 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:41.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 5 16 15842 15826 197.791 131.25 0.00283099 0.00504916 2024-04-10T03:09:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:42 smithi152 ceph-mon[33523]: pgmap v8: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 62 MiB/s wr, 1.01k op/s 2024-04-10T03:09:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:42 smithi102 ceph-mon[39651]: pgmap v8: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 62 MiB/s wr, 1.01k op/s 2024-04-10T03:09:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:42 smithi102 ceph-mon[94535]: pgmap v8: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 62 MiB/s wr, 1.01k op/s 2024-04-10T03:09:42.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 6 16 18607 18591 193.625 172.812 0.00672391 0.00515871 2024-04-10T03:09:43.176 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 podman[83608]: 2024-04-10 03:09:42.923401447 +0000 UTC m=+0.031863305 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:09:43.176 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 podman[83608]: 2024-04-10 03:09:43.037952056 +0000 UTC m=+0.146413905 container create 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64) 2024-04-10T03:09:43.464 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 podman[83608]: 2024-04-10 03:09:43.358240619 +0000 UTC m=+0.466702483 container init 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, distribution-scope=public, com.redhat.component=ubi8-container, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=) 2024-04-10T03:09:43.464 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 podman[83608]: 2024-04-10 03:09:43.363536097 +0000 UTC m=+0.471997943 container start 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, version=8.5, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, architecture=x86_64, distribution-scope=public, maintainer=Paul Cuzner , vcs-type=git, io.buildah.version=1.32.0) 2024-04-10T03:09:43.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 bash[83608]: 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f 2024-04-10T03:09:43.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:43.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[83649]: logger=settings t=2024-04-10T03:09:43.660309387Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:09:43.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:43 smithi152 podman[85312]: 2024-04-10 03:09:43.708352574 +0000 UTC m=+0.036582364 container died 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , name=ubi8) 2024-04-10T03:09:43.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 7 16 21250 21234 189.561 165.188 0.00367915 0.00526924 2024-04-10T03:09:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:43 smithi152 ceph-mon[33523]: pgmap v9: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 48 MiB/s wr, 783 op/s 2024-04-10T03:09:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:43 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:43 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[39651]: pgmap v9: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 48 MiB/s wr, 783 op/s 2024-04-10T03:09:44.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[94535]: pgmap v9: 89 pgs: 89 active+clean; 434 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 48 MiB/s wr, 783 op/s 2024-04-10T03:09:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:43 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:44.873 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 8 16 24065 24049 187.856 175.938 0.0101753 0.00531786 2024-04-10T03:09:45.011 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:44 smithi152 podman[85312]: 2024-04-10 03:09:44.600848835 +0000 UTC m=+0.929078622 container remove 8acdc1bc2d0f32175281e15764777c3dd3c2428bad243fa3274bb5c6d3c7ce7f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , architecture=x86_64) 2024-04-10T03:09:45.011 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:44 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:09:45.511 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:45 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:09:45.511 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:45 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.060s CPU time. 2024-04-10T03:09:45.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 9 16 26341 26325 182.788 142.25 0.00747864 0.00546525 2024-04-10T03:09:46.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 126 MiB/s wr, 2.03k op/s 2024-04-10T03:09:46.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:46.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:46.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:09:46.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:46 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 126 MiB/s wr, 2.03k op/s 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[94535]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:09:46.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: pgmap v10: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 126 MiB/s wr, 2.03k op/s 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.conf 2024-04-10T03:09:46.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:46 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-10T03:09:46.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 10 16 28503 28487 178.02 135.125 0.00675612 0.00561147 2024-04-10T03:09:47.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:47 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.conf 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: Updating smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.544 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:47 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:47.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 11 15 30473 30458 173.035 123.188 0.00895109 0.00577385 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 109 MiB/s wr, 1.76k op/s 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: pgmap v12: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 127 MiB/s wr, 2.04k op/s 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: pgmap v13: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s rd, 154 MiB/s wr, 2.48k op/s 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:48.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 109 MiB/s wr, 1.76k op/s 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: pgmap v12: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 127 MiB/s wr, 2.04k op/s 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: pgmap v13: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s rd, 154 MiB/s wr, 2.48k op/s 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: pgmap v11: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 17 KiB/s rd, 109 MiB/s wr, 1.76k op/s 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: pgmap v12: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 127 MiB/s wr, 2.04k op/s 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: pgmap v13: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s rd, 154 MiB/s wr, 2.48k op/s 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:09:48.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:48 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:09:48.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 12 16 32755 32739 170.495 142.562 0.00405357 0.00585946 2024-04-10T03:09:49.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:09:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:09:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:09:49.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 13 16 35120 35104 168.749 147.812 0.00516796 0.00591944 2024-04-10T03:09:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:50 smithi152 ceph-mon[33523]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:50 smithi102 ceph-mon[39651]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:50 smithi102 ceph-mon[94535]: pgmap v14: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:50.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 14 16 37406 37390 166.9 142.875 0.00652614 0.00598559 2024-04-10T03:09:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:09:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:09:50] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:09:51.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 15 15 39618 39603 164.993 138.312 0.00852094 0.00605624 2024-04-10T03:09:52.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:52 smithi152 ceph-mon[33523]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:52.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:52 smithi102 ceph-mon[94535]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:52.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:52 smithi102 ceph-mon[39651]: pgmap v15: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2024-04-10T03:09:52.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 16 16 41709 41693 162.845 130.625 0.0105596 0.00613552 2024-04-10T03:09:53.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 17 16 43791 43775 160.919 130.125 0.00456563 0.0062096 2024-04-10T03:09:54.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:54 smithi152 ceph-mon[33523]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:09:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:54 smithi102 ceph-mon[39651]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:09:54.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:54 smithi102 ceph-mon[94535]: pgmap v16: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-10T03:09:54.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 18 16 45805 45789 158.972 125.875 0.0108071 0.00628528 2024-04-10T03:09:55.819 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-10T03:09:55.819 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:55.819 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.060s CPU time. 2024-04-10T03:09:55.820 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 systemd[1]: Starting Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:09:55.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 19 16 47776 47760 157.088 123.188 0.0128085 0.0063609 2024-04-10T03:09:56.151 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 podman[86854]: 2024-04-10 03:09:55.724872997 +0000 UTC m=+0.029627397 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-10T03:09:56.163 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:55 smithi152 podman[86854]: 2024-04-10 03:09:55.850827374 +0000 UTC m=+0.155581757 container create ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vcs-type=git, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, release=239.1651231664, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, maintainer=Paul Cuzner ) 2024-04-10T03:09:56.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:56 smithi152 ceph-mon[33523]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-10T03:09:56.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 podman[86854]: 2024-04-10 03:09:56.153881023 +0000 UTC m=+0.458635423 container init ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, version=8.5) 2024-04-10T03:09:56.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 podman[86854]: 2024-04-10 03:09:56.158216051 +0000 UTC m=+0.462970435 container start ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, architecture=x86_64, io.buildah.version=1.32.0, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-10T03:09:56.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 bash[86854]: ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e 2024-04-10T03:09:56.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 systemd[1]: Started Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:09:56.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a[86870]: logger=settings t=2024-04-10T03:09:56.425001774Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-10T03:09:56.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:56 smithi152 podman[86887]: 2024-04-10 03:09:56.475125662 +0000 UTC m=+0.035318557 container died ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, name=ubi8, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.buildah.version=1.32.0, release=239.1651231664, vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-10T03:09:56.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:56 smithi102 ceph-mon[39651]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-10T03:09:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:56 smithi102 ceph-mon[94535]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-10T03:09:56.874 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:09:56.878285+0000 min lat: 0.00127348 max lat: 0.168105 avg lat: 0.00643707 2024-04-10T03:09:56.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:09:56.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 20 16 49693 49677 155.224 119.812 0.0103762 0.00643707 2024-04-10T03:09:57.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:57 smithi152 podman[86887]: 2024-04-10 03:09:57.052540868 +0000 UTC m=+0.612733763 container remove ada57f5737951047e328c04e9ad8d6572f66fd26edf0d4faf54cba4268a70c9e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, version=8.5, name=ubi8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git) 2024-04-10T03:09:57.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:57 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-10T03:09:57.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:57 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:09:57.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:09:57 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.022s CPU time. 2024-04-10T03:09:57.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 21 16 51480 51464 153.15 111.688 0.0121241 0.00652424 2024-04-10T03:09:58.657 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:09:58 smithi152 ceph-mon[33523]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:09:58.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:09:58 smithi102 ceph-mon[39651]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:09:58.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:09:58 smithi102 ceph-mon[94535]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:09:58.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 22 16 53328 53312 151.438 115.5 0.00806071 0.00659868 2024-04-10T03:09:59.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:09:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:09:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:09:59.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 23 16 55128 55112 149.745 112.5 0.00524912 0.00667331 2024-04-10T03:10:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:00 smithi152 ceph-mon[33523]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-04-10T03:10:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:00 smithi152 ceph-mon[33523]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:10:00.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:00 smithi102 ceph-mon[39651]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-04-10T03:10:00.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:00 smithi102 ceph-mon[39651]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:10:00.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:00 smithi102 ceph-mon[94535]: pgmap v19: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 167 MiB/s wr, 2.68k op/s 2024-04-10T03:10:00.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:00 smithi102 ceph-mon[94535]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:10:00.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 24 16 56894 56878 148.104 110.375 0.0109286 0.00674677 2024-04-10T03:10:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:00] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:10:01.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 25 16 58603 58587 146.452 106.812 0.0131955 0.00682313 2024-04-10T03:10:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:02 smithi152 ceph-mon[33523]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:02.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:02 smithi102 ceph-mon[39651]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:02.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:02 smithi102 ceph-mon[94535]: pgmap v20: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:02.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 26 16 60218 60202 144.701 100.938 0.00714636 0.00690618 2024-04-10T03:10:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:03 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:03.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:03.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 27 16 61881 61865 143.191 103.938 0.0100023 0.0069784 2024-04-10T03:10:04.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:04 smithi152 ceph-mon[33523]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:04.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:04 smithi102 ceph-mon[39651]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:04.692 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:04 smithi102 ceph-mon[94535]: pgmap v21: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-10T03:10:04.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 28 16 63506 63490 141.704 101.562 0.0162755 0.00705162 2024-04-10T03:10:05.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 29 16 65066 65050 140.18 97.5 0.0181691 0.00712817 2024-04-10T03:10:06.394 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:06 smithi102 ceph-mon[94535]: pgmap v22: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-10T03:10:06.394 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:06 smithi102 ceph-mon[39651]: pgmap v22: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-10T03:10:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:06 smithi152 ceph-mon[33523]: pgmap v22: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-10T03:10:06.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 30 16 66431 66415 138.351 85.3125 0.00804391 0.00722212 2024-04-10T03:10:07.761 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 10. 2024-04-10T03:10:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: Stopped Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:10:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Consumed 1.022s CPU time. 2024-04-10T03:10:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Start request repeated too quickly. 2024-04-10T03:10:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-10T03:10:07.762 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Apr 10 03:10:07 smithi152 systemd[1]: Failed to start Ceph grafana.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:10:07.875 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 31 16 67827 67811 136.702 87.25 0.0113601 0.00731001 2024-04-10T03:10:08.657 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:08 smithi152 ceph-mon[33523]: pgmap v23: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-10T03:10:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:08 smithi102 ceph-mon[94535]: pgmap v23: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-10T03:10:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:08 smithi102 ceph-mon[39651]: pgmap v23: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-10T03:10:08.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 32 16 69348 69332 135.4 95.0625 0.00727474 0.00738025 2024-04-10T03:10:09.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:09.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 33 16 70813 70797 134.072 91.5625 0.0148681 0.00745316 2024-04-10T03:10:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:10 smithi152 ceph-mon[33523]: pgmap v24: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-10T03:10:10.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:10 smithi102 ceph-mon[39651]: pgmap v24: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-10T03:10:10.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:10 smithi102 ceph-mon[94535]: pgmap v24: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-04-10T03:10:10.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 34 16 72283 72267 132.83 91.875 0.00680762 0.00752356 2024-04-10T03:10:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:10] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.43.0" 2024-04-10T03:10:11.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 35 16 73768 73752 131.687 92.8125 0.00789769 0.00758898 2024-04-10T03:10:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:12 smithi152 ceph-mon[33523]: pgmap v25: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:12 smithi102 ceph-mon[39651]: pgmap v25: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:12 smithi102 ceph-mon[94535]: pgmap v25: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:12.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 36 16 75200 75184 130.515 89.5 0.00847772 0.00765641 2024-04-10T03:10:13.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 37 16 76633 76617 129.408 89.5625 0.0050703 0.00772251 2024-04-10T03:10:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:14 smithi152 ceph-mon[33523]: pgmap v26: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:14.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:14 smithi102 ceph-mon[39651]: pgmap v26: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:14.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:14 smithi102 ceph-mon[94535]: pgmap v26: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-04-10T03:10:14.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 38 16 78041 78025 128.318 88 0.00750428 0.00778836 2024-04-10T03:10:15.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 39 16 79395 79379 127.197 84.625 0.00860391 0.00785661 2024-04-10T03:10:16.691 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:16 smithi152 ceph-mon[33523]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-04-10T03:10:16.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:16 smithi102 ceph-mon[39651]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-04-10T03:10:16.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:16 smithi102 ceph-mon[94535]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-04-10T03:10:16.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:10:16.880030+0000 min lat: 0.00127348 max lat: 0.168105 avg lat: 0.00792987 2024-04-10T03:10:16.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:10:16.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 40 16 80685 80669 126.033 80.625 0.0128253 0.00792987 2024-04-10T03:10:17.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 41 16 81979 81963 124.931 80.875 0.0215766 0.00799924 2024-04-10T03:10:18.657 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:18 smithi152 ceph-mon[33523]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-10T03:10:18.657 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:18 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:18.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:18 smithi102 ceph-mon[39651]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-10T03:10:18.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:18 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:18 smithi102 ceph-mon[94535]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-10T03:10:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:18.876 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 42 16 83289 83273 123.906 81.875 0.0134344 0.00806529 2024-04-10T03:10:19.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:19.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 43 16 84599 84583 122.928 81.875 0.0195527 0.00812947 2024-04-10T03:10:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:20 smithi152 ceph-mon[33523]: pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-10T03:10:20.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:20 smithi102 ceph-mon[39651]: pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-10T03:10:20.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:20 smithi102 ceph-mon[94535]: pgmap v29: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-10T03:10:20.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 44 16 85900 85884 121.982 81.3125 0.0161727 0.00819293 2024-04-10T03:10:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:20] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:10:21.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 45 16 87152 87136 121.01 78.25 0.0114877 0.00825782 2024-04-10T03:10:22.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:22 smithi152 ceph-mon[33523]: pgmap v30: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:22 smithi102 ceph-mon[39651]: pgmap v30: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:22 smithi102 ceph-mon[94535]: pgmap v30: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:22.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 46 16 88444 88428 120.135 80.75 0.0222868 0.00831898 2024-04-10T03:10:23.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 47 16 89705 89689 119.256 78.8125 0.0134663 0.00838029 2024-04-10T03:10:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:24 smithi152 ceph-mon[33523]: pgmap v31: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:24 smithi102 ceph-mon[39651]: pgmap v31: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:24.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:24 smithi102 ceph-mon[94535]: pgmap v31: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-10T03:10:24.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 48 16 90924 90908 118.358 76.1875 0.0123141 0.00844374 2024-04-10T03:10:25.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 49 16 92155 92139 117.513 76.9375 0.0159043 0.00850475 2024-04-10T03:10:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:26 smithi152 ceph-mon[33523]: pgmap v32: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-10T03:10:26.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:26 smithi102 ceph-mon[39651]: pgmap v32: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-10T03:10:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:26 smithi102 ceph-mon[94535]: pgmap v32: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-10T03:10:26.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 50 16 93395 93379 116.712 77.5 0.00518548 0.00856355 2024-04-10T03:10:27.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 51 16 94596 94580 115.896 75.0625 0.0143488 0.00862303 2024-04-10T03:10:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:28 smithi152 ceph-mon[33523]: pgmap v33: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-10T03:10:28.762 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:28.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:28 smithi102 ceph-mon[39651]: pgmap v33: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-10T03:10:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:28 smithi102 ceph-mon[94535]: pgmap v33: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-10T03:10:28.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 52 16 95796 95780 115.109 75 0.0081387 0.00868252 2024-04-10T03:10:29.877 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 53 16 96991 96975 114.346 74.6875 0.0184803 0.00874007 2024-04-10T03:10:30.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:30 smithi152 ceph-mon[33523]: pgmap v34: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-10T03:10:30.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:30 smithi102 ceph-mon[39651]: pgmap v34: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-10T03:10:30.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:30 smithi102 ceph-mon[94535]: pgmap v34: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-10T03:10:30.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 54 16 98185 98169 113.611 74.625 0.00611167 0.00879691 2024-04-10T03:10:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:30] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:10:31.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 55 16 99361 99345 112.881 73.5 0.00629079 0.00885364 2024-04-10T03:10:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:32 smithi152 ceph-mon[33523]: pgmap v35: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:32 smithi102 ceph-mon[39651]: pgmap v35: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:32 smithi102 ceph-mon[94535]: pgmap v35: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:32.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 56 16 100480 100464 112.114 69.9375 0.00602584 0.00891468 2024-04-10T03:10:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:33 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:33.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 57 16 101614 101598 111.391 70.875 0.00792253 0.00897182 2024-04-10T03:10:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:34 smithi152 ceph-mon[33523]: pgmap v36: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:34.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:34 smithi102 ceph-mon[94535]: pgmap v36: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:34.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:34 smithi102 ceph-mon[39651]: pgmap v36: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-10T03:10:34.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 58 16 102755 102739 110.699 71.3125 0.0157775 0.00902842 2024-04-10T03:10:35.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 59 16 103881 103865 110.016 70.375 0.0184977 0.00908501 2024-04-10T03:10:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:36 smithi152 ceph-mon[33523]: pgmap v37: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-04-10T03:10:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:36 smithi102 ceph-mon[94535]: pgmap v37: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-04-10T03:10:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:36 smithi102 ceph-mon[39651]: pgmap v37: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-04-10T03:10:36.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:10:36.881849+0000 min lat: 0.00127348 max lat: 0.168105 avg lat: 0.00913976 2024-04-10T03:10:36.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:10:36.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 60 16 105008 104992 109.356 70.4375 0.00767628 0.00913976 2024-04-10T03:10:37.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 61 16 106111 106095 108.693 68.9375 0.0111582 0.00919557 2024-04-10T03:10:38.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:38 smithi152 ceph-mon[33523]: pgmap v38: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-10T03:10:38.762 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:38 smithi102 ceph-mon[94535]: pgmap v38: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-10T03:10:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:38 smithi102 ceph-mon[39651]: pgmap v38: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-04-10T03:10:38.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 62 16 107208 107192 108.046 68.5625 0.0126136 0.00925034 2024-04-10T03:10:39.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 63 16 108293 108277 107.407 67.8125 0.0322384 0.00930509 2024-04-10T03:10:40.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:40 smithi152 ceph-mon[33523]: pgmap v39: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-04-10T03:10:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:40 smithi102 ceph-mon[94535]: pgmap v39: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-04-10T03:10:40.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:40 smithi102 ceph-mon[39651]: pgmap v39: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-04-10T03:10:40.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 64 16 109366 109350 106.777 67.0625 0.0150746 0.00936068 2024-04-10T03:10:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:40] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2024-04-10T03:10:41.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 65 16 110431 110415 106.158 66.5625 0.0136201 0.00941471 2024-04-10T03:10:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:42 smithi152 ceph-mon[33523]: pgmap v40: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:42 smithi102 ceph-mon[39651]: pgmap v40: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:42.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:42 smithi102 ceph-mon[94535]: pgmap v40: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:42.878 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 66 16 111505 111489 105.567 67.125 0.0182655 0.00946727 2024-04-10T03:10:43.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 67 16 112557 112541 104.972 65.75 0.0146405 0.00952068 2024-04-10T03:10:44.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:44 smithi102 ceph-mon[94535]: pgmap v41: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:44.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:44 smithi102 ceph-mon[39651]: pgmap v41: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:44.740 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade resume' 2024-04-10T03:10:44.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:44 smithi152 ceph-mon[33523]: pgmap v41: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-04-10T03:10:44.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 68 16 113560 113544 104.35 62.6875 0.0228471 0.00957811 2024-04-10T03:10:45.245 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:10:45.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 69 16 114502 114486 103.691 58.875 0.010405 0.0096388 2024-04-10T03:10:46.721 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:46 smithi102 ceph-mon[39651]: pgmap v42: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-10T03:10:46.721 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:46 smithi102 ceph-mon[94535]: pgmap v42: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-10T03:10:46.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:46 smithi152 ceph-mon[33523]: pgmap v42: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-10T03:10:46.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 70 16 115497 115481 103.098 62.1875 0.0157918 0.00969388 2024-04-10T03:10:47.692 INFO:teuthology.orchestra.run.smithi102.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:10:47.692 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:47 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:10:47.692 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:47 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:10:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:47 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:10:47.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 71 16 116328 116312 102.378 51.9375 0.0258367 0.00976239 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[39651]: pgmap v43: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[94535]: pgmap v43: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:48.755 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:48 smithi152 ceph-mon[33523]: pgmap v43: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-04-10T03:10:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:48 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:10:48.761 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:48.787 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-10T03:10:48.880 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 72 16 117240 117224 101.747 57 0.0267612 0.00982357 2024-04-10T03:10:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:49 smithi152 ceph-mon[33523]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:10:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:49 smithi152 ceph-mon[33523]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:10:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:49 smithi102 ceph-mon[94535]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:10:49.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:49 smithi102 ceph-mon[94535]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:10:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:49 smithi102 ceph-mon[39651]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:10:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:49 smithi102 ceph-mon[39651]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:10:49.879 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 73 16 118121 118105 101.108 55.0625 0.0157296 0.00988556 2024-04-10T03:10:49.947 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:10:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:50 smithi102 ceph-mon[39651]: pgmap v44: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-04-10T03:10:50.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:50 smithi102 ceph-mon[94535]: pgmap v44: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-04-10T03:10:50.882 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 74 16 119035 119019 100.512 57.125 0.014379 0.00994364 2024-04-10T03:10:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:50 smithi152 ceph-mon[33523]: pgmap v44: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-04-10T03:10:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:10:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:10:50] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2024-04-10T03:10:51.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 75 16 119872 119856 99.8689 52.3125 0.0158925 0.0100004 2024-04-10T03:10:52.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:52 smithi152 ceph-mon[33523]: pgmap v45: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:52.774 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:52 smithi102 ceph-mon[94535]: pgmap v45: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:52.774 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:52 smithi102 ceph-mon[39651]: pgmap v45: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:52.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 76 16 120624 120608 99.1732 47 0.00794943 0.0100774 2024-04-10T03:10:53.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 77 16 121523 121507 98.6149 56.1875 0.00572959 0.0101318 2024-04-10T03:10:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[94535]: pgmap v46: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:54.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:54 smithi102 ceph-mon[39651]: pgmap v46: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:54 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:54 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:10:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:54 smithi152 ceph-mon[33523]: pgmap v46: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2024-04-10T03:10:54.742 INFO:teuthology.orchestra.run.smithi102.stdout:wait for majority of mons to upgrade 2024-04-10T03:10:54.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 78 16 122404 122388 98.0565 55.0625 0.00594449 0.010193 2024-04-10T03:10:55.508 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-10T03:10:55.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 79 16 123286 123270 97.5131 55.125 0.0135825 0.0102492 2024-04-10T03:10:56.458 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:10:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:56 smithi102 ceph-mon[94535]: pgmap v47: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-10T03:10:56.545 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:56 smithi102 ceph-mon[39651]: pgmap v47: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-10T03:10:56.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:56 smithi152 ceph-mon[33523]: pgmap v47: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-04-10T03:10:56.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:10:56.884838+0000 min lat: 0.00127348 max lat: 0.168105 avg lat: 0.0103049 2024-04-10T03:10:56.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:10:56.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 80 16 124184 124168 96.9957 56.125 0.0251948 0.0103049 2024-04-10T03:10:57.881 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 81 16 125116 125100 96.5172 58.25 0.0115799 0.0103556 2024-04-10T03:10:58.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:58 smithi152 ceph-mon[33523]: pgmap v48: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2024-04-10T03:10:58.664 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:58 smithi102 ceph-mon[94535]: pgmap v48: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2024-04-10T03:10:58.666 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:58 smithi102 ceph-mon[39651]: pgmap v48: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2024-04-10T03:10:58.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 82 16 126002 125986 96.0145 55.375 0.0157638 0.0104099 2024-04-10T03:10:59.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:10:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:10:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:10:59.882 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 83 16 126805 126789 95.4623 50.1875 0.0161183 0.0104704 2024-04-10T03:11:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:59 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:59 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:10:59 smithi152 ceph-mon[33523]: pgmap v49: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:11:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[94535]: pgmap v49: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:11:00.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:00.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:10:59 smithi102 ceph-mon[39651]: pgmap v49: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-10T03:11:00.882 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 84 16 127576 127560 94.8995 48.1875 0.0282404 0.010532 2024-04-10T03:11:01.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:01.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:01.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: pgmap v50: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: pgmap v51: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: Upgrade: It appears safe to stop mon.c 2024-04-10T03:11:01.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:00 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3622721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:00] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: pgmap v50: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: pgmap v51: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: Upgrade: It appears safe to stop mon.c 2024-04-10T03:11:01.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/3622721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: pgmap v50: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: pgmap v51: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: Upgrade: It appears safe to stop mon.c 2024-04-10T03:11:01.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:00 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3622721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:01.882 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 85 16 128450 128434 94.4256 54.625 0.0164063 0.0105851 2024-04-10T03:11:02.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 86 16 129343 129327 93.9761 55.8125 0.0120042 0.0106354 2024-04-10T03:11:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: pgmap v52: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:11:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:11:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: pgmap v52: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:03.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: pgmap v52: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-04-10T03:11:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: from='mgr.25253 ' entity='mgr.y' 2024-04-10T03:11:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:11:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:11:03.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:03 smithi102 ceph-mon[39651]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:03.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 87 16 130077 130061 93.4232 45.875 0.0301205 0.0106983 2024-04-10T03:11:04.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:04 smithi152 ceph-mon[33523]: Upgrade: Updating mon.c 2024-04-10T03:11:04.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:04 smithi152 ceph-mon[33523]: Deploying daemon mon.c on smithi102 2024-04-10T03:11:04.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:04 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/2064968929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[94535]: Upgrade: Updating mon.c 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[94535]: Deploying daemon mon.c on smithi102 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2064968929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[39651]: Upgrade: Updating mon.c 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[39651]: Deploying daemon mon.c on smithi102 2024-04-10T03:11:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:04 smithi102 ceph-mon[39651]: from='client.? 172.21.15.102:0/2064968929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:04.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 88 16 130904 130888 92.9488 51.6875 0.0186035 0.0107529 2024-04-10T03:11:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:05 smithi152 ceph-mon[33523]: pgmap v53: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-04-10T03:11:05.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:05 smithi102 ceph-mon[94535]: pgmap v53: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-04-10T03:11:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:05 smithi102 ceph-mon[39651]: pgmap v53: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-04-10T03:11:05.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:05 smithi102 systemd[1]: Stopping Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:11:05.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 89 16 131569 131553 92.3714 41.5625 0.0344306 0.0108205 2024-04-10T03:11:06.461 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c[39646]: 2024-04-10T03:11:06.080+0000 7f2565e16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c[39646]: 2024-04-10T03:11:06.080+0000 7f2565e16700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 podman[98614]: 2024-04-10 03:11:06.239668887 +0000 UTC m=+0.316604172 container died 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE Bus STOPPING 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE Bus STOPPED 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE Bus STARTING 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE Serving on http://:::9283 2024-04-10T03:11:06.462 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:06 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:11:06] ENGINE Bus STARTED 2024-04-10T03:11:06.723 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 podman[98614]: 2024-04-10 03:11:06.709771609 +0000 UTC m=+0.786706896 container cleanup 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:11:06.723 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 bash[98614]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c 2024-04-10T03:11:06.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total time run: 90.0106 2024-04-10T03:11:06.883 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total writes made: 132365 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Write size: 65536 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object size: 65536 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Bandwidth (MB/sec): 91.9093 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Bandwidth: 41.0324 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max bandwidth (MB/sec): 248.438 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min bandwidth (MB/sec): 41.5625 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average IOPS: 1470 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev IOPS: 656.518 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max IOPS: 3975 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min IOPS: 665 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average Latency(s): 0.0108759 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Latency(s): 0.00761189 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max latency(s): 0.168105 2024-04-10T03:11:06.884 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min latency(s): 0.00127348 2024-04-10T03:11:06.889 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Cleaning up (deleting benchmark objects) 2024-04-10T03:11:07.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:06 smithi102 podman[98636]: 2024-04-10 03:11:06.92019177 +0000 UTC m=+0.673768187 container remove 8f8b8b481e8d866335826823ef98e369141493c4ecc269b61ec8d09b4b772a15 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-10T03:11:07.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:07 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.c.service: Deactivated successfully. 2024-04-10T03:11:07.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:07 smithi102 systemd[1]: Stopped Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:11:07.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:07 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.c.service: Consumed 8.963s CPU time. 2024-04-10T03:11:08.033 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:07 smithi102 systemd[1]: Starting Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:11:08.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 podman[98755]: 2024-04-10 03:11:07.942252497 +0000 UTC m=+0.043414913 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:11:08.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 podman[98755]: 2024-04-10 03:11:08.071593115 +0000 UTC m=+0.172755527 container create ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:11:08.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 podman[98755]: 2024-04-10 03:11:08.342110356 +0000 UTC m=+0.443272782 container init ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:11:08.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 podman[98755]: 2024-04-10 03:11:08.348773226 +0000 UTC m=+0.449935645 container start ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: set uid:gid to 167:167 (ceph:ceph) 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev), process ceph-mon, pid 2 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: pidfile_write: ignore empty --pid-file 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: load: jerasure load: lrc 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: RocksDB version: 7.9.2 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Git sha 0 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Compile date 2024-04-09 01:35:31 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: DB SUMMARY 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: DB Session ID: WTJLUXMGX5BZL83WHICH 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: CURRENT file: CURRENT 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: IDENTITY file: IDENTITY 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1093 Bytes 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-10T03:11:08.797 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 5322560 ; 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.error_if_exists: 0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.create_if_missing: 0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.paranoid_checks: 1 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.env: 0x559bdd753c60 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.fs: PosixFileSystem 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.info_log: 0x559bdf159ae0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_file_opening_threads: 16 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.statistics: (nil) 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.use_fsync: 0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_log_file_size: 0 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-10T03:11:08.798 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.keep_log_file_num: 1000 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.recycle_log_file_num: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_fallocate: 1 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_mmap_reads: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_mmap_writes: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.use_direct_reads: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.create_missing_column_families: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.db_log_dir: 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.wal_dir: 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-10T03:11:08.799 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.advise_random_on_open: 1 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.db_write_buffer_size: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.write_buffer_manager: 0x559bdf12c820 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.rate_limiter: (nil) 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.wal_recovery_mode: 2 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enable_thread_tracking: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enable_pipelined_write: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.unordered_write: 0 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-10T03:11:08.800 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.row_cache: None 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.wal_filter: None 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_ingest_behind: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.two_write_queues: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.manual_wal_flush: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.wal_compression: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.atomic_flush: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.log_readahead_size: 0 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-10T03:11:08.801 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.best_efforts_recovery: 0 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.allow_data_in_errors: 0 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.db_host_id: __hostname__ 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_background_jobs: 2 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_background_compactions: -1 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_subcompactions: 1 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_total_wal_size: 0 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-10T03:11:08.802 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_open_files: -1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bytes_per_sync: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_readahead_size: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_background_flushes: -1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Compression algorithms supported: 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kZSTD supported: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kXpressCompression supported: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kLZ4HCCompression supported: 1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kZlibCompression supported: 1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kSnappyCompression supported: 1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kLZ4Compression supported: 1 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: kBZip2Compression supported: 0 2024-04-10T03:11:08.803 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.merge_operator: 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_filter: None 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_filter_factory: None 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.sst_partitioner_factory: None 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559bdf159be0) 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_type: 0 2024-04-10T03:11:08.804 INFO:journalctl@ceph.mon.c.smithi102.stdout: data_block_index_type: 0 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_shortening: 1 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: checksum: 4 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: no_block_cache: 0 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache: 0x559bdf15d1f0 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_options: 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: capacity : 536870912 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: num_shard_bits : 4 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: strict_capacity_limit : 0 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_cache_compressed: (nil) 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: persistent_cache: (nil) 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_size: 4096 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_size_deviation: 10 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_restart_interval: 16 2024-04-10T03:11:08.805 INFO:journalctl@ceph.mon.c.smithi102.stdout: index_block_restart_interval: 1 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: metadata_block_size: 4096 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: partition_filters: 0 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: use_delta_encoding: 1 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: filter_policy: bloomfilter 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: whole_key_filtering: 1 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: verify_compression: 0 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: format_version: 5 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: enable_index_compression: 1 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: block_align: 0 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: max_auto_readahead_size: 262144 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: prepopulate_block_cache: 0 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: initial_auto_readahead_size: 8192 2024-04-10T03:11:08.806 INFO:journalctl@ceph.mon.c.smithi102.stdout: num_file_reads_for_auto_readahead: 2 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.write_buffer_size: 33554432 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_write_buffer_number: 2 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression: NoCompression 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression: Disabled 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.prefix_extractor: nullptr 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.num_levels: 7 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:11:08.807 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.level: 32767 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.strategy: 0 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.enabled: false 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-10T03:11:08.808 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.target_file_size_base: 67108864 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.arena_block_size: 1048576 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-10T03:11:08.809 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.disable_auto_compactions: 0 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.table_properties_collectors: 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.inplace_update_support: 0 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-10T03:11:08.810 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.bloom_locality: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.max_successive_merges: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.paranoid_file_checks: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.force_consistency_checks: 1 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.report_bg_io_stats: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.ttl: 2592000 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enable_blob_files: false 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.min_blob_size: 0 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_file_size: 268435456 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-10T03:11:08.811 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.blob_file_starting_level: 0 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12747, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-10T03:11:08.812 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8e20b443-dae5-40e8-a2f8-6b427d9a5a70 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718668399950, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718668421431, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3361256, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12744, "largest_seqno": 14721, "table_properties": {"data_size": 3352676, "index_size": 5576, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1989, "raw_key_size": 17712, "raw_average_key_size": 22, "raw_value_size": 3335787, "raw_average_value_size": 4271, "num_data_blocks": 252, "num_entries": 781, "num_filter_entries": 781, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712718668, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8e20b443-dae5-40e8-a2f8-6b427d9a5a70", "db_session_id": "WTJLUXMGX5BZL83WHICH", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718668421618, "job": 1, "event": "recovery_finished"} 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559bdf24a000 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: DB pointer 0x559bdf238000 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** DB Stats ** 2024-04-10T03:11:08.813 INFO:journalctl@ceph.mon.c.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: L0 1/0 3.21 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 327.5 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: L6 1/0 7.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Sum 2/0 10.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 327.5 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-10T03:11:08.814 INFO:journalctl@ceph.mon.c.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 327.5 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** Compaction Stats [default] ** 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 327.5 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T03:11:08.815 INFO:journalctl@ceph.mon.c.smithi102.stdout: Cumulative compaction: 0.00 GB write, 115.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: Interval compaction: 0.00 GB write, 115.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: Block cache BinnedLRUCache@0x559bdf15d1f0#2 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.8e-05 secs_since: 0 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: starting mon.c rank 2 at public addrs [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] at bind addrs [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???) e3 preinit fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).mds e9 new map 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).mds e9 print_map 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: e9 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: legacy client fscid: 1 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.816 INFO:journalctl@ceph.mon.c.smithi102.stdout: Filesystem 'foo' (1) 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: fs_name foo 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: epoch 8 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: created 2024-04-10T03:03:52.164573+0000 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: modified 2024-04-10T03:04:03.738495+0000 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: tableserver 0 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: root 0 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: session_timeout 60 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: session_autoclose 300 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: max_file_size 1099511627776 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: max_xattr_size 65536 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: required_client_features {} 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: last_failure 0 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: last_failure_osd_epoch 0 2024-04-10T03:11:08.817 INFO:journalctl@ceph.mon.c.smithi102.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: max_mds 1 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: in 0 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: up {0=24545} 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: failed 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: damaged 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: stopped 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: data_pools [4] 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: metadata_pool 3 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: inline_data disabled 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: balancer 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: bal_rank_mask -1 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: standby_count_wanted 1 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: qdb_cluster leader: 0 members: 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: [mds.foo.smithi102.exkvif{0:24545} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:11:08.818 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout: Standby daemons: 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout: 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout: [mds.foo.smithi152.adssru{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 ceph-mon[98773]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 bash[98755]: ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe 2024-04-10T03:11:08.819 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:08 smithi102 systemd[1]: Started Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:11:09.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:10] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-04-10T03:11:14.779 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 2 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: mon.c calling monitor election 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: mon.b calling monitor election 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: mon.a calling monitor election 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: monmap epoch 3 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: min_mon_release 17 (quincy) 2024-04-10T03:11:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: election_strategy: 1 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: mgrmap e40: y(active, since 101s), standbys: x 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:11:15.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:14 smithi152 ceph-mon[33523]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:11:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: mon.c calling monitor election 2024-04-10T03:11:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: mon.b calling monitor election 2024-04-10T03:11:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: mon.a calling monitor election 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: monmap epoch 3 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: min_mon_release 17 (quincy) 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: election_strategy: 1 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:11:15.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: mgrmap e40: y(active, since 101s), standbys: x 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[98773]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: mon.c calling monitor election 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: mon.b calling monitor election 2024-04-10T03:11:15.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: mon.a calling monitor election 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: monmap epoch 3 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: last_changed 2024-04-10T02:58:36.178392+0000 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: min_mon_release 17 (quincy) 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: election_strategy: 1 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:11:15.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:11:15.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: mgrmap e40: y(active, since 101s), standbys: x 2024-04-10T03:11:15.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-10T03:11:15.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:11:15.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:11:15.045 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:14 smithi102 ceph-mon[94535]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:11:15.983 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4011445255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:15.984 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4011445255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:16.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:15 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/4011445255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:11:16.339 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:11:16.841 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:11:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:16 smithi152 ceph-mon[33523]: pgmap v59: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:17.036 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:16 smithi102 ceph-mon[98773]: pgmap v59: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:17.036 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:16 smithi102 ceph-mon[94535]: pgmap v59: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:18.131 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:18.131 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:18.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:17 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:18.696 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:18 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:11:18.443+0000 7f1b6ff70700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-10T03:11:19.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:19.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:19 smithi152 ceph-mon[33523]: pgmap v60: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:19 smithi102 ceph-mon[94535]: pgmap v60: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:19.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:19 smithi102 ceph-mon[98773]: pgmap v60: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2024-04-10T03:11:19.894 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:11:19.894 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (3m) 21s ago 8m 24.2M - 0.25.0 c8568f914cd2 65832d2b0608 2024-04-10T03:11:19.894 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 error 26s ago 8m - - 2024-04-10T03:11:19.894 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (7m) 21s ago 7m 15.3M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (7m) 26s ago 7m 15.6M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443,9283,8765 running (2m) 26s ago 12m 449M - 19.0.0-2051-g3a866975 06a486fd9bae 10f3bdd94f98 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:8443,9283,8765 running (3m) 21s ago 14m 497M - 19.0.0-2051-g3a866975 06a486fd9bae ba38a088e643 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (117s) 21s ago 14m 37.2M 2048M 19.0.0-2051-g3a866975 06a486fd9bae c3949a498743 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 running (12m) 26s ago 12m 50.0M 2048M 17.2.7-827-gd3b5060a 21de9b7369ad 7ecfd799af6d 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 starting - - - 2048M 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (3m) 21s ago 8m 16.0M - 1.5.0 0da6a335fe13 773a048f8daf 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (3m) 26s ago 7m 16.8M - 1.5.0 0da6a335fe13 6010c1e76260 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (12m) 21s ago 12m 357M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 984949346b35 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (11m) 21s ago 11m 304M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 13648c191186 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (11m) 21s ago 11m 419M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad f10dfba3072f 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (10m) 21s ago 10m 433M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 6f7249bbf2b8 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (10m) 26s ago 10m 378M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (10m) 26s ago 10m 407M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:11:19.895 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (9m) 26s ago 9m 273M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:11:19.896 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (9m) 26s ago 9m 350M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:11:19.896 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (2m) 26s ago 8m 56.4M - 2.43.0 a07b618ecd1d a7900639fc5e 2024-04-10T03:11:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:20] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2024-04-10T03:11:21.643 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:21 smithi102 ceph-mon[98773]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:21.643 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:21 smithi102 ceph-mon[98773]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-10T03:11:21.643 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:21 smithi102 ceph-mon[94535]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:21.643 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:21 smithi102 ceph-mon[94535]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-10T03:11:21.684 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade pause' 2024-04-10T03:11:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:21 smithi152 ceph-mon[33523]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:21 smithi152 ceph-mon[33523]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-10T03:11:22.534 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:11:23.959 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[94535]: pgmap v62: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-04-10T03:11:23.959 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:23.959 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:23.960 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[98773]: pgmap v62: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-04-10T03:11:23.960 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:23.960 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:23 smithi152 ceph-mon[33523]: pgmap v62: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-04-10T03:11:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:23 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:23 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:25.319 INFO:teuthology.orchestra.run.smithi102.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:11:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:25 smithi152 ceph-mon[33523]: pgmap v63: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:25 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:25 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:25 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[98773]: pgmap v63: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[94535]: pgmap v63: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:26.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:26.103 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'sleep 60' 2024-04-10T03:11:26.714 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:26 smithi102 ceph-mon[98773]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:26.714 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:26 smithi102 ceph-mon[98773]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:11:26.714 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:26 smithi102 ceph-mon[94535]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:26.714 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:26 smithi102 ceph-mon[94535]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:11:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:26 smithi152 ceph-mon[33523]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:11:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:26 smithi152 ceph-mon[33523]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:11:27.109 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:11:27.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:27 smithi102 ceph-mon[94535]: pgmap v64: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:27.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:27 smithi102 ceph-mon[98773]: pgmap v64: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:27 smithi152 ceph-mon[33523]: pgmap v64: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:28.694 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:28 smithi102 ceph-mon[98773]: pgmap v65: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:28.694 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:28 smithi102 ceph-mon[94535]: pgmap v65: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:28 smithi152 ceph-mon[33523]: pgmap v65: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-04-10T03:11:29.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:31.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:30] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2024-04-10T03:11:31.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:31 smithi152 ceph-mon[33523]: pgmap v66: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:11:31.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:31 smithi102 ceph-mon[94535]: pgmap v66: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:11:31.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:31 smithi102 ceph-mon[98773]: pgmap v66: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:11:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: pgmap v67: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:11:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: pgmap v67: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: pgmap v67: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:11:33.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:11:34.693 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:34 smithi102 ceph-mon[98773]: pgmap v68: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-04-10T03:11:34.694 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:34 smithi102 ceph-mon[98773]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2024-04-10T03:11:34.694 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:34 smithi102 ceph-mon[94535]: pgmap v68: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-04-10T03:11:34.694 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:34 smithi102 ceph-mon[94535]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2024-04-10T03:11:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:34 smithi152 ceph-mon[33523]: pgmap v68: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-04-10T03:11:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:34 smithi152 ceph-mon[33523]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2024-04-10T03:11:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:36 smithi152 ceph-mon[33523]: pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:36 smithi102 ceph-mon[98773]: pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:36 smithi102 ceph-mon[94535]: pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:38.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:38 smithi152 ceph-mon[33523]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:38.763 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:38 smithi102 ceph-mon[98773]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:38 smithi102 ceph-mon[94535]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-10T03:11:40.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:40 smithi152 ceph-mon[33523]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-04-10T03:11:40.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:40 smithi102 ceph-mon[98773]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-04-10T03:11:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:40 smithi102 ceph-mon[94535]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-04-10T03:11:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:40] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2024-04-10T03:11:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:42 smithi152 ceph-mon[33523]: pgmap v73: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:11:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:42 smithi102 ceph-mon[98773]: pgmap v73: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:11:42.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:42 smithi102 ceph-mon[94535]: pgmap v73: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-10T03:11:44.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:44 smithi152 ceph-mon[33523]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-04-10T03:11:44.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:44 smithi102 ceph-mon[98773]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-04-10T03:11:44.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:44 smithi102 ceph-mon[94535]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-04-10T03:11:46.718 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:46 smithi102 ceph-mon[98773]: pgmap v75: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:46.719 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:46 smithi102 ceph-mon[94535]: pgmap v75: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:46.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:46 smithi152 ceph-mon[33523]: pgmap v75: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-10T03:11:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:48 smithi152 ceph-mon[33523]: pgmap v76: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-10T03:11:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:48.761 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:48.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:48 smithi102 ceph-mon[98773]: pgmap v76: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-10T03:11:48.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:48 smithi102 ceph-mon[94535]: pgmap v76: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-10T03:11:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:11:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:50 smithi152 ceph-mon[33523]: pgmap v77: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:11:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:50 smithi102 ceph-mon[98773]: pgmap v77: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:11:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:50 smithi102 ceph-mon[94535]: pgmap v77: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:11:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:11:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:11:50] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2024-04-10T03:11:52.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:52 smithi152 ceph-mon[33523]: pgmap v78: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-04-10T03:11:52.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:52 smithi102 ceph-mon[98773]: pgmap v78: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-04-10T03:11:52.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:52 smithi102 ceph-mon[94535]: pgmap v78: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-04-10T03:11:54.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:54 smithi152 ceph-mon[33523]: pgmap v79: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:11:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:54 smithi102 ceph-mon[98773]: pgmap v79: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:11:54.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:54 smithi102 ceph-mon[94535]: pgmap v79: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:11:56.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:56 smithi152 ceph-mon[33523]: pgmap v80: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:11:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:56 smithi102 ceph-mon[94535]: pgmap v80: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:11:56.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:56 smithi102 ceph-mon[98773]: pgmap v80: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:11:58.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:11:58 smithi152 ceph-mon[33523]: pgmap v81: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-04-10T03:11:58.761 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:11:58 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:11:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:11:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:11:58 smithi102 ceph-mon[94535]: pgmap v81: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-04-10T03:11:58.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:11:58 smithi102 ceph-mon[98773]: pgmap v81: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-04-10T03:12:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:00 smithi152 ceph-mon[33523]: pgmap v82: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:12:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:00 smithi102 ceph-mon[94535]: pgmap v82: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:12:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:00 smithi102 ceph-mon[98773]: pgmap v82: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:12:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:00] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2024-04-10T03:12:02.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:02 smithi102 ceph-mon[94535]: pgmap v83: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-04-10T03:12:02.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:02 smithi102 ceph-mon[98773]: pgmap v83: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-04-10T03:12:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:02 smithi152 ceph-mon[33523]: pgmap v83: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-04-10T03:12:03.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:03 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:04.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:04 smithi102 ceph-mon[94535]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:04.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:04 smithi102 ceph-mon[98773]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:04.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:04 smithi152 ceph-mon[33523]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:06.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:06 smithi102 ceph-mon[98773]: pgmap v85: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:06.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:06 smithi102 ceph-mon[94535]: pgmap v85: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:06 smithi152 ceph-mon[33523]: pgmap v85: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:08.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:08 smithi102 ceph-mon[98773]: pgmap v86: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:08 smithi102 ceph-mon[94535]: pgmap v86: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:08 smithi152 ceph-mon[33523]: pgmap v86: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-04-10T03:12:09.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:12:08 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:12:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:12:10.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:10 smithi102 ceph-mon[94535]: pgmap v87: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:10.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:10 smithi102 ceph-mon[98773]: pgmap v87: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:10 smithi152 ceph-mon[33523]: pgmap v87: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:10] "GET /metrics HTTP/1.1" 200 36225 "" "Prometheus/2.43.0" 2024-04-10T03:12:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:12 smithi152 ceph-mon[33523]: pgmap v88: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-04-10T03:12:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:12 smithi102 ceph-mon[94535]: pgmap v88: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-04-10T03:12:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:12 smithi102 ceph-mon[98773]: pgmap v88: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-04-10T03:12:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:14 smithi152 ceph-mon[33523]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:12:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:14 smithi102 ceph-mon[98773]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:12:15.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:14 smithi102 ceph-mon[94535]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-10T03:12:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:16 smithi152 ceph-mon[33523]: pgmap v90: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-10T03:12:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:16 smithi102 ceph-mon[94535]: pgmap v90: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-10T03:12:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:16 smithi102 ceph-mon[98773]: pgmap v90: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-04-10T03:12:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:18 smithi152 ceph-mon[33523]: pgmap v91: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-04-10T03:12:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:18 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:19.012 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:12:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:12:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:12:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:18 smithi102 ceph-mon[98773]: pgmap v91: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-04-10T03:12:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:19.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:18 smithi102 ceph-mon[94535]: pgmap v91: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s 2024-04-10T03:12:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:20.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:20 smithi102 ceph-mon[98773]: pgmap v92: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-04-10T03:12:20.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:20 smithi102 ceph-mon[94535]: pgmap v92: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-04-10T03:12:21.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:20 smithi152 ceph-mon[33523]: pgmap v92: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-04-10T03:12:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:20] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2024-04-10T03:12:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:22 smithi152 ceph-mon[33523]: pgmap v93: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-04-10T03:12:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:22 smithi102 ceph-mon[98773]: pgmap v93: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-04-10T03:12:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:22 smithi102 ceph-mon[94535]: pgmap v93: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-04-10T03:12:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:24 smithi152 ceph-mon[33523]: pgmap v94: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-10T03:12:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:24 smithi102 ceph-mon[94535]: pgmap v94: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-10T03:12:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:24 smithi102 ceph-mon[98773]: pgmap v94: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s 2024-04-10T03:12:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:26 smithi152 ceph-mon[33523]: pgmap v95: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:26 smithi102 ceph-mon[94535]: pgmap v95: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:27.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:26 smithi102 ceph-mon[98773]: pgmap v95: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-10T03:12:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:28 smithi152 ceph-mon[33523]: pgmap v96: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:29.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:12:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:12:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:12:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:28 smithi102 ceph-mon[94535]: pgmap v96: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:29.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:28 smithi102 ceph-mon[98773]: pgmap v96: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:30.917 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:30 smithi102 ceph-mon[94535]: pgmap v97: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:12:30.917 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:30 smithi102 ceph-mon[98773]: pgmap v97: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:12:30.959 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade resume' 2024-04-10T03:12:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:30 smithi152 ceph-mon[33523]: pgmap v97: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-10T03:12:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:30] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2024-04-10T03:12:31.440 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:12:32.918 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:32 smithi102 ceph-mon[98773]: pgmap v98: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-04-10T03:12:32.919 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:32 smithi102 ceph-mon[94535]: pgmap v98: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-04-10T03:12:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:32 smithi152 ceph-mon[33523]: pgmap v98: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-04-10T03:12:33.845 INFO:teuthology.orchestra.run.smithi102.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:12:33.869 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:33.869 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:33.869 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:33.869 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:33 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:34.572 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'echo wait for all mons to upgrade' 2024-04-10T03:12:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:34 smithi152 ceph-mon[33523]: pgmap v99: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:34 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:34 smithi102 ceph-mon[94535]: pgmap v99: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:34 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:34.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:34 smithi102 ceph-mon[98773]: pgmap v99: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-04-10T03:12:34.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:34 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:35.629 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:12:35.989 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:35 smithi102 ceph-mon[94535]: from='client.25159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:35.989 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:35 smithi102 ceph-mon[94535]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:12:35.989 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:35 smithi102 ceph-mon[98773]: from='client.25159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:35.989 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:35 smithi102 ceph-mon[98773]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:12:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:35 smithi152 ceph-mon[33523]: from='client.25159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:35 smithi152 ceph-mon[33523]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:12:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:36 smithi152 ceph-mon[33523]: pgmap v100: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-10T03:12:37.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:36 smithi102 ceph-mon[94535]: pgmap v100: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-10T03:12:37.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:36 smithi102 ceph-mon[98773]: pgmap v100: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-10T03:12:37.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:37 smithi152 ceph-mon[33523]: pgmap v101: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-04-10T03:12:37.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:37 smithi102 ceph-mon[98773]: pgmap v101: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-04-10T03:12:37.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:37 smithi102 ceph-mon[94535]: pgmap v101: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-04-10T03:12:39.011 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:12:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:12:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)\", cluster_addr=\"172.21.15.102\", device_class=\"ssd\", hostname=\"smithi102\", instance=\"smithi102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.102\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-10T03:12:39.459 INFO:teuthology.orchestra.run.smithi102.stdout:wait for all mons to upgrade 2024-04-10T03:12:40.207 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-10T03:12:40.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:40 smithi102 ceph-mon[98773]: pgmap v102: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-04-10T03:12:40.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:40 smithi102 ceph-mon[94535]: pgmap v102: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-04-10T03:12:40.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:40 smithi152 ceph-mon[33523]: pgmap v102: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-04-10T03:12:41.273 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:40] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2024-04-10T03:12:41.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:41 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:41.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:41 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:41.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:41 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:41.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:41 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:41.739 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:12:41.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:41 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:41.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:41 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:42 smithi152 ceph-mon[33523]: pgmap v103: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-10T03:12:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:42 smithi102 ceph-mon[98773]: pgmap v103: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-10T03:12:42.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:42 smithi102 ceph-mon[94535]: pgmap v103: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-10T03:12:44.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:44 smithi102 ceph-mon[94535]: pgmap v104: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:12:44.693 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:44 smithi102 ceph-mon[98773]: pgmap v104: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:12:44.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:44 smithi152 ceph-mon[33523]: pgmap v104: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-10T03:12:46.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/3850991016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:46.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:45 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3850991016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3850991016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:45 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: pgmap v106: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-10T03:12:46.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:46 smithi152 ceph-mon[33523]: Upgrade: It appears safe to stop mon.b 2024-04-10T03:12:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: pgmap v106: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[94535]: Upgrade: It appears safe to stop mon.b 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: pgmap v106: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-10T03:12:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:46 smithi102 ceph-mon[98773]: Upgrade: It appears safe to stop mon.b 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: pgmap v107: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: Upgrade: Updating mon.b 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:48.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: Deploying daemon mon.b on smithi152 2024-04-10T03:12:48.918 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:48 smithi152 ceph-mon[33523]: from='client.? 172.21.15.102:0/1161955928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: pgmap v107: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: Upgrade: Updating mon.b 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: Deploying daemon mon.b on smithi152 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1161955928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: pgmap v107: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: Upgrade: Updating mon.b 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:12:49.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:12:49.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:12:49.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: Deploying daemon mon.b on smithi152 2024-04-10T03:12:49.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:48 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1161955928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:50.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 systemd[1]: Stopping Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:12:50.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b[33511]: 2024-04-10T03:12:50.533+0000 7f67fd95d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:12:50.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b[33511]: 2024-04-10T03:12:50.533+0000 7f67fd95d700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-10T03:12:50.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 podman[88496]: 2024-04-10 03:12:50.56532974 +0000 UTC m=+0.160184006 container died 7ecfd799af6d41ee8186ca93030e17a86c1877939bf40921812d9a2735b5d628 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409) 2024-04-10T03:12:51.237 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 podman[88496]: 2024-04-10 03:12:50.975112333 +0000 UTC m=+0.569966582 container cleanup 7ecfd799af6d41ee8186ca93030e17a86c1877939bf40921812d9a2735b5d628 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-10T03:12:51.237 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:50 smithi152 bash[88496]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b 2024-04-10T03:12:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:12:50] "GET /metrics HTTP/1.1" 200 36226 "" "Prometheus/2.43.0" 2024-04-10T03:12:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:51 smithi152 podman[88510]: 2024-04-10 03:12:51.24051412 +0000 UTC m=+0.671183932 container remove 7ecfd799af6d41ee8186ca93030e17a86c1877939bf40921812d9a2735b5d628 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:12:51.856 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:51 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.b.service: Deactivated successfully. 2024-04-10T03:12:51.856 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:51 smithi152 systemd[1]: Stopped Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:12:51.856 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:51 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.b.service: Consumed 11.421s CPU time. 2024-04-10T03:12:52.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:51 smithi152 systemd[1]: Starting Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:12:52.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 podman[88623]: 2024-04-10 03:12:52.073786865 +0000 UTC m=+0.028551531 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:12:52.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 podman[88623]: 2024-04-10 03:12:52.20450023 +0000 UTC m=+0.159264895 container create 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:12:52.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 podman[88623]: 2024-04-10 03:12:52.460228335 +0000 UTC m=+0.414993000 container init 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-10T03:12:52.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 podman[88623]: 2024-04-10 03:12:52.464845077 +0000 UTC m=+0.419609748 container start 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: set uid:gid to 167:167 (ceph:ceph) 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev), process ceph-mon, pid 2 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: pidfile_write: ignore empty --pid-file 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: load: jerasure load: lrc 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: RocksDB version: 7.9.2 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Git sha 0 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Compile date 2024-04-09 01:35:31 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: DB SUMMARY 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: DB Session ID: 015T954N1GBO4HBE1IJY 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: CURRENT file: CURRENT 2024-04-10T03:12:52.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: IDENTITY file: IDENTITY 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1093 Bytes 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 7048108 ; 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.error_if_exists: 0 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.create_if_missing: 0 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.paranoid_checks: 1 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.env: 0x557202c8bc60 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.fs: PosixFileSystem 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.info_log: 0x55720484dae0 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_file_opening_threads: 16 2024-04-10T03:12:52.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.statistics: (nil) 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.use_fsync: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_log_file_size: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.keep_log_file_num: 1000 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.recycle_log_file_num: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_fallocate: 1 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_mmap_reads: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_mmap_writes: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.use_direct_reads: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.create_missing_column_families: 0 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.db_log_dir: 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.wal_dir: 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-10T03:12:52.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.advise_random_on_open: 1 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.db_write_buffer_size: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.write_buffer_manager: 0x557204820820 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.rate_limiter: (nil) 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.wal_recovery_mode: 2 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enable_thread_tracking: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enable_pipelined_write: 0 2024-04-10T03:12:52.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.unordered_write: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.row_cache: None 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.wal_filter: None 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_ingest_behind: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.two_write_queues: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.manual_wal_flush: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.wal_compression: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.atomic_flush: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-10T03:12:52.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.log_readahead_size: 0 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.best_efforts_recovery: 0 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.allow_data_in_errors: 0 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.db_host_id: __hostname__ 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_background_jobs: 2 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_background_compactions: -1 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_subcompactions: 1 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-10T03:12:52.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_total_wal_size: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_open_files: -1 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bytes_per_sync: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_readahead_size: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_background_flushes: -1 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Compression algorithms supported: 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kZSTD supported: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kXpressCompression supported: 0 2024-04-10T03:12:52.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kLZ4HCCompression supported: 1 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kZlibCompression supported: 1 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kSnappyCompression supported: 1 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kLZ4Compression supported: 1 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: kBZip2Compression supported: 0 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.merge_operator: 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_filter: None 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_filter_factory: None 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.sst_partitioner_factory: None 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-10T03:12:52.774 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55720484dbe0) 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_type: 0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_index_type: 0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_shortening: 1 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: checksum: 4 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: no_block_cache: 0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache: 0x5572048511f0 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_options: 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: capacity : 536870912 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_shard_bits : 4 2024-04-10T03:12:52.775 INFO:journalctl@ceph.mon.b.smithi152.stdout: strict_capacity_limit : 0 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_compressed: (nil) 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: persistent_cache: (nil) 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size: 4096 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size_deviation: 10 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_restart_interval: 16 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_block_restart_interval: 1 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_block_size: 4096 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: partition_filters: 0 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: use_delta_encoding: 1 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: filter_policy: bloomfilter 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: whole_key_filtering: 1 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: verify_compression: 0 2024-04-10T03:12:52.776 INFO:journalctl@ceph.mon.b.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: format_version: 5 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_index_compression: 1 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_align: 0 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_auto_readahead_size: 262144 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: prepopulate_block_cache: 0 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: initial_auto_readahead_size: 8192 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_file_reads_for_auto_readahead: 2 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.write_buffer_size: 33554432 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_write_buffer_number: 2 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression: NoCompression 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression: Disabled 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.prefix_extractor: nullptr 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.num_levels: 7 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-10T03:12:52.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.level: 32767 2024-04-10T03:12:52.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.strategy: 0 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.enabled: false 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.target_file_size_base: 67108864 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-10T03:12:52.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.arena_block_size: 1048576 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-10T03:12:52.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.disable_auto_compactions: 0 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.table_properties_collectors: 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.inplace_update_support: 0 2024-04-10T03:12:52.781 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.bloom_locality: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.max_successive_merges: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.paranoid_file_checks: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.force_consistency_checks: 1 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.report_bg_io_stats: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.ttl: 2592000 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enable_blob_files: false 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.min_blob_size: 0 2024-04-10T03:12:52.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_file_size: 268435456 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.blob_file_starting_level: 0 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12820, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 4978ab3c-84f2-434a-915c-406931ce45bd 2024-04-10T03:12:52.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718772508263, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718772533752, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4398296, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12817, "largest_seqno": 16181, "table_properties": {"data_size": 4385176, "index_size": 8515, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3589, "raw_key_size": 30015, "raw_average_key_size": 21, "raw_value_size": 4356926, "raw_average_value_size": 3087, "num_data_blocks": 384, "num_entries": 1411, "num_filter_entries": 1411, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712718772, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "4978ab3c-84f2-434a-915c-406931ce45bd", "db_session_id": "015T954N1GBO4HBE1IJY", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712718772533894, "job": 1, "event": "recovery_finished"} 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557204942000 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: DB pointer 0x55720492c000 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** DB Stats ** 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:12:52.784 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: L0 1/0 4.19 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: L6 1/0 7.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Sum 2/0 11.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-10T03:12:52.785 INFO:journalctl@ceph.mon.b.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 396.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative compaction: 0.00 GB write, 121.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval compaction: 0.00 GB write, 121.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache BinnedLRUCache@0x5572048511f0#2 capacity: 512.00 MB usage: 69.03 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2024-04-10T03:12:52.786 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache entry stats(count,size,portion): DataBlock(4,30.81 KB,0.00587702%) FilterBlock(2,13.06 KB,0.00249147%) IndexBlock(2,25.16 KB,0.00479817%) Misc(1,0.00 KB,0%) 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: starting mon.b rank 1 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???) e3 preinit fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).mds e9 new map 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).mds e9 print_map 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: e9 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: legacy client fscid: 1 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: Filesystem 'foo' (1) 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: fs_name foo 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: epoch 8 2024-04-10T03:12:52.787 INFO:journalctl@ceph.mon.b.smithi152.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: created 2024-04-10T03:03:52.164573+0000 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: modified 2024-04-10T03:04:03.738495+0000 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: tableserver 0 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: root 0 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_timeout 60 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_autoclose 300 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_file_size 1099511627776 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_xattr_size 65536 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: required_client_features {} 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure 0 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure_osd_epoch 0 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-10T03:12:52.788 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_mds 1 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: in 0 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: up {0=24545} 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: failed 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: damaged 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: stopped 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_pools [4] 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_pool 3 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: inline_data disabled 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: balancer 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: bal_rank_mask -1 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: standby_count_wanted 1 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: qdb_cluster leader: 0 members: 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi102.exkvif{0:24545} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.102:6834/2927235058,v1:172.21.15.102:6835/2927235058] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.789 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout: Standby daemons: 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi152.adssru{-1:24517} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/4181242426,v1:172.21.15.152:6833/4181242426] compat {c=[1],r=[1],i=[7ff]}] 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 ceph-mon[88640]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 bash[88623]: 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 2024-04-10T03:12:52.790 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:52 smithi152 systemd[1]: Started Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:12:53.709 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 3 2024-04-10T03:12:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:53 smithi152 ceph-mon[88640]: pgmap v108: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-04-10T03:12:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:53 smithi102 ceph-mon[94535]: pgmap v108: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-04-10T03:12:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:53 smithi102 ceph-mon[98773]: pgmap v108: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-04-10T03:12:54.437 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: pgmap v110: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mon.b calling monitor election 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mon.c calling monitor election 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mon.a calling monitor election 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:12:54.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: monmap epoch 4 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: last_changed 2024-04-10T03:12:53.628710+0000 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: min_mon_release 19 (squid) 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: election_strategy: 1 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3467774235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:54.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:54.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: pgmap v110: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mon.b calling monitor election 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mon.c calling monitor election 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mon.a calling monitor election 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: monmap epoch 4 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: last_changed 2024-04-10T03:12:53.628710+0000 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: min_mon_release 19 (squid) 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: election_strategy: 1 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:54.795 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3467774235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:54.796 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:54.876 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:12:55.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: pgmap v110: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mon.b calling monitor election 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mon.c calling monitor election 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mon.a calling monitor election 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: monmap epoch 4 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: last_changed 2024-04-10T03:12:53.628710+0000 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: created 2024-04-10T02:56:57.710085+0000 2024-04-10T03:12:55.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: min_mon_release 19 (squid) 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: election_strategy: 1 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: 0: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.a 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: 2: [v2:172.21.15.102:3301/0,v1:172.21.15.102:6790/0] mon.c 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: fsmap foo:1 {0=foo.smithi102.exkvif=up:active} 1 up:standby 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: osdmap e82: 8 total, 8 up, 8 in 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3467774235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-10T03:12:55.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-10T03:12:55.015 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:55.015 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:12:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:56 smithi152 ceph-mon[88640]: pgmap v111: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-04-10T03:12:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:56 smithi102 ceph-mon[98773]: pgmap v111: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-04-10T03:12:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:56 smithi102 ceph-mon[94535]: pgmap v111: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-04-10T03:12:57.254 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:12:57.254 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (5m) 12s ago 10m 24.2M - 0.25.0 c8568f914cd2 65832d2b0608 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 error 16s ago 9m - - 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (8m) 12s ago 8m 16.2M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (8m) 16s ago 8m 16.0M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443,9283,8765 running (3m) 16s ago 14m 450M - 19.0.0-2051-g3a866975 06a486fd9bae 10f3bdd94f98 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:8443,9283,8765 running (4m) 12s ago 15m 501M - 19.0.0-2051-g3a866975 06a486fd9bae ba38a088e643 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (3m) 12s ago 15m 41.6M 2048M 19.0.0-2051-g3a866975 06a486fd9bae c3949a498743 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 starting - - - 2048M 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 running (109s) 12s ago 14m 31.7M 2048M 19.0.0-2051-g3a866975 06a486fd9bae ff269d657b5a 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (5m) 12s ago 9m 16.3M - 1.5.0 0da6a335fe13 773a048f8daf 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (4m) 16s ago 9m 17.2M - 1.5.0 0da6a335fe13 6010c1e76260 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (13m) 12s ago 13m 381M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 984949346b35 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (13m) 12s ago 13m 341M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 13648c191186 2024-04-10T03:12:57.255 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (13m) 12s ago 12m 477M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad f10dfba3072f 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (12m) 12s ago 12m 499M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 6f7249bbf2b8 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (12m) 16s ago 12m 411M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (11m) 16s ago 11m 433M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (11m) 16s ago 11m 310M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (10m) 16s ago 10m 393M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:12:57.256 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (4m) 16s ago 10m 58.9M - 2.43.0 a07b618ecd1d a7900639fc5e 2024-04-10T03:12:57.966 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:12:57 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:12:57.554+0000 7f1b6ff70700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-10T03:12:58.322 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade pause' 2024-04-10T03:12:58.775 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:12:58.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:58 smithi102 ceph-mon[94535]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:58.924 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:58 smithi102 ceph-mon[98773]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:58 smithi152 ceph-mon[88640]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[98773]: pgmap v112: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[94535]: pgmap v112: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:12:59.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:12:59 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:59 smithi152 ceph-mon[88640]: pgmap v112: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-10T03:13:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:59 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:12:59 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.157 INFO:teuthology.orchestra.run.smithi102.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:13:01.190 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:00] "GET /metrics HTTP/1.1" 200 36226 "" "Prometheus/2.43.0" 2024-04-10T03:13:01.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:01 smithi152 ceph-mon[88640]: pgmap v113: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:01.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:01 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:01 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:01 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.535 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[98773]: pgmap v113: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:01.535 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.535 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.535 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.536 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[94535]: pgmap v113: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:01.536 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.536 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:01.536 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:01 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:02.118 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'sleep 60' 2024-04-10T03:13:02.601 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:13:02.744 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:02 smithi102 ceph-mon[98773]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:13:02.744 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:02 smithi102 ceph-mon[98773]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:13:02.745 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:02 smithi102 ceph-mon[94535]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:13:02.745 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:02 smithi102 ceph-mon[94535]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:13:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:02 smithi152 ceph-mon[88640]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:13:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:02 smithi152 ceph-mon[88640]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:13:03.667 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:03 smithi102 ceph-mon[98773]: pgmap v114: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:03.667 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:03.667 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:03 smithi102 ceph-mon[94535]: pgmap v114: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:03.667 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:03 smithi152 ceph-mon[88640]: pgmap v114: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:05.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:05 smithi102 ceph-mon[98773]: pgmap v115: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:05.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:05 smithi102 ceph-mon[94535]: pgmap v115: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:05.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:05 smithi152 ceph-mon[88640]: pgmap v115: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: pgmap v116: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: pgmap v117: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.726 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: Reconfiguring mon.a (monmap changed)... 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[94535]: Reconfiguring daemon mon.a on smithi102 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: pgmap v116: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: pgmap v117: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: Reconfiguring mon.a (monmap changed)... 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:06.727 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:06.728 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:06.728 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:06 smithi102 ceph-mon[98773]: Reconfiguring daemon mon.a on smithi102 2024-04-10T03:13:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: pgmap v116: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: pgmap v117: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: Reconfiguring mon.a (monmap changed)... 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:07.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:06 smithi152 ceph-mon[88640]: Reconfiguring daemon mon.a on smithi102 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: Reconfiguring mgr.y (monmap changed)... 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:09.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:09 smithi152 ceph-mon[88640]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: Reconfiguring mgr.y (monmap changed)... 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[98773]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: pgmap v118: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: Reconfiguring mgr.y (monmap changed)... 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:09.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:09 smithi102 ceph-mon[94535]: Reconfiguring daemon mgr.y on smithi102 2024-04-10T03:13:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:10] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-04-10T03:13:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:11 smithi152 ceph-mon[88640]: pgmap v119: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:11 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:11.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:11 smithi102 ceph-mon[94535]: pgmap v119: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:11.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:11 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:11.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:11 smithi102 ceph-mon[98773]: pgmap v119: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:11.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:11 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: Reconfiguring mon.c (monmap changed)... 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[98773]: Reconfiguring daemon mon.c on smithi102 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: Reconfiguring mon.c (monmap changed)... 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:12.739 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:12 smithi102 ceph-mon[94535]: Reconfiguring daemon mon.c on smithi102 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: Reconfiguring mon.c (monmap changed)... 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:12 smithi152 ceph-mon[88640]: Reconfiguring daemon mon.c on smithi102 2024-04-10T03:13:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:13 smithi152 ceph-mon[88640]: pgmap v120: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:13.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:13 smithi102 ceph-mon[94535]: pgmap v120: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:13.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:13 smithi102 ceph-mon[98773]: pgmap v120: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:14.910 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:14.910 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:14.910 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: pgmap v121: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:14.910 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: Reconfiguring osd.0 (monmap changed)... 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.0 on smithi102 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: pgmap v121: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: Reconfiguring osd.0 (monmap changed)... 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:14.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:14 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.0 on smithi102 2024-04-10T03:13:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: pgmap v121: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-04-10T03:13:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: Reconfiguring osd.0 (monmap changed)... 2024-04-10T03:13:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:13:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:14 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.0 on smithi102 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: pgmap v122: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: Reconfiguring osd.1 (monmap changed)... 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.1 on smithi102 2024-04-10T03:13:17.480 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: pgmap v122: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: Reconfiguring osd.1 (monmap changed)... 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:17.481 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:17 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.1 on smithi102 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: pgmap v122: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: Reconfiguring osd.1 (monmap changed)... 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:17 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.1 on smithi102 2024-04-10T03:13:18.426 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:18.427 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: pgmap v123: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:13:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: Reconfiguring osd.2 (monmap changed)... 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.2 on smithi102 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: pgmap v123: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: Reconfiguring osd.2 (monmap changed)... 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:19.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:19 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.2 on smithi102 2024-04-10T03:13:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: pgmap v123: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-10T03:13:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: Reconfiguring osd.2 (monmap changed)... 2024-04-10T03:13:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:13:20.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:20.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:19 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.2 on smithi102 2024-04-10T03:13:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:20] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[98773]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[94535]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:21.923 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:13:21.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:21 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:21 smithi152 ceph-mon[88640]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-10T03:13:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:21 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:21 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:22.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:21 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:13:22.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:21 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:22.982 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:22 smithi102 ceph-mon[94535]: Reconfiguring osd.3 (monmap changed)... 2024-04-10T03:13:22.982 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:22 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.3 on smithi102 2024-04-10T03:13:22.983 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:22 smithi102 ceph-mon[98773]: Reconfiguring osd.3 (monmap changed)... 2024-04-10T03:13:22.983 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:22 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.3 on smithi102 2024-04-10T03:13:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:22 smithi152 ceph-mon[88640]: Reconfiguring osd.3 (monmap changed)... 2024-04-10T03:13:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:22 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.3 on smithi102 2024-04-10T03:13:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:23 smithi152 ceph-mon[88640]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[98773]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[94535]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi102.exkvif", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:24 smithi102 ceph-mon[98773]: Reconfiguring mds.foo.smithi102.exkvif (monmap changed)... 2024-04-10T03:13:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:24 smithi102 ceph-mon[98773]: Reconfiguring daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:13:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:24 smithi102 ceph-mon[94535]: Reconfiguring mds.foo.smithi102.exkvif (monmap changed)... 2024-04-10T03:13:24.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:24 smithi102 ceph-mon[94535]: Reconfiguring daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:13:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:24 smithi152 ceph-mon[88640]: Reconfiguring mds.foo.smithi102.exkvif (monmap changed)... 2024-04-10T03:13:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:24 smithi152 ceph-mon[88640]: Reconfiguring daemon mds.foo.smithi102.exkvif on smithi102 2024-04-10T03:13:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: pgmap v126: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-10T03:13:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:26 smithi152 ceph-mon[88640]: Reconfiguring mon.b (monmap changed)... 2024-04-10T03:13:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:26 smithi152 ceph-mon[88640]: Reconfiguring daemon mon.b on smithi152 2024-04-10T03:13:27.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:26 smithi102 ceph-mon[98773]: Reconfiguring mon.b (monmap changed)... 2024-04-10T03:13:27.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:26 smithi102 ceph-mon[98773]: Reconfiguring daemon mon.b on smithi152 2024-04-10T03:13:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:26 smithi102 ceph-mon[94535]: Reconfiguring mon.b (monmap changed)... 2024-04-10T03:13:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:26 smithi102 ceph-mon[94535]: Reconfiguring daemon mon.b on smithi152 2024-04-10T03:13:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:27 smithi152 ceph-mon[88640]: pgmap v127: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:27 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:27 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[98773]: pgmap v127: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[94535]: pgmap v127: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-10T03:13:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:27 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:28.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:28 smithi152 ceph-mon[88640]: Reconfiguring mgr.x (monmap changed)... 2024-04-10T03:13:28.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:28 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:28.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:28 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:28.748 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:28 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[98773]: Reconfiguring mgr.x (monmap changed)... 2024-04-10T03:13:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[94535]: Reconfiguring mgr.x (monmap changed)... 2024-04-10T03:13:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-10T03:13:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-10T03:13:29.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:28 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:29 smithi152 ceph-mon[88640]: Reconfiguring daemon mgr.x on smithi152 2024-04-10T03:13:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:29 smithi152 ceph-mon[88640]: pgmap v128: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-04-10T03:13:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:29 smithi102 ceph-mon[98773]: Reconfiguring daemon mgr.x on smithi152 2024-04-10T03:13:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:29 smithi102 ceph-mon[98773]: pgmap v128: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-04-10T03:13:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:29 smithi102 ceph-mon[94535]: Reconfiguring daemon mgr.x on smithi152 2024-04-10T03:13:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:29 smithi102 ceph-mon[94535]: pgmap v128: 89 pgs: 89 active+clean; 938 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: pgmap v129: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: Reconfiguring osd.4 (monmap changed)... 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:30.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:30 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.4 on smithi152 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: pgmap v129: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: Reconfiguring osd.4 (monmap changed)... 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.4 on smithi152 2024-04-10T03:13:30.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: pgmap v129: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: Reconfiguring osd.4 (monmap changed)... 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:30.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.4 on smithi152 2024-04-10T03:13:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:30] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2024-04-10T03:13:32.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: pgmap v130: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: Reconfiguring osd.5 (monmap changed)... 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:32.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:32 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.5 on smithi152 2024-04-10T03:13:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: pgmap v130: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: Reconfiguring osd.5 (monmap changed)... 2024-04-10T03:13:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.5 on smithi152 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: pgmap v130: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: Reconfiguring osd.5 (monmap changed)... 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:32 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.5 on smithi152 2024-04-10T03:13:34.163 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:34.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:34.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:35 smithi152 ceph-mon[88640]: pgmap v131: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:35 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:35 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:35.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:35 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:13:35.930 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:35 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[98773]: pgmap v131: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:13:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[94535]: pgmap v131: 89 pgs: 89 active+clean; 668 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-10T03:13:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-10T03:13:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:35 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:36 smithi152 ceph-mon[88640]: Reconfiguring osd.6 (monmap changed)... 2024-04-10T03:13:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:36 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.6 on smithi152 2024-04-10T03:13:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:36 smithi102 ceph-mon[98773]: Reconfiguring osd.6 (monmap changed)... 2024-04-10T03:13:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:36 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.6 on smithi152 2024-04-10T03:13:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:36 smithi102 ceph-mon[94535]: Reconfiguring osd.6 (monmap changed)... 2024-04-10T03:13:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:36 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.6 on smithi152 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[98773]: pgmap v132: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:37.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[94535]: pgmap v132: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:13:37.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:37.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:37.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:13:37.935 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:37 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:37 smithi152 ceph-mon[88640]: pgmap v132: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-10T03:13:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:37 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:37 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:37 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-10T03:13:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:37 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:38 smithi152 ceph-mon[88640]: Reconfiguring osd.7 (monmap changed)... 2024-04-10T03:13:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:38 smithi152 ceph-mon[88640]: Reconfiguring daemon osd.7 on smithi152 2024-04-10T03:13:39.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:38 smithi102 ceph-mon[98773]: Reconfiguring osd.7 (monmap changed)... 2024-04-10T03:13:39.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:38 smithi102 ceph-mon[98773]: Reconfiguring daemon osd.7 on smithi152 2024-04-10T03:13:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:38 smithi102 ceph-mon[94535]: Reconfiguring osd.7 (monmap changed)... 2024-04-10T03:13:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:38 smithi102 ceph-mon[94535]: Reconfiguring daemon osd.7 on smithi152 2024-04-10T03:13:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: pgmap v133: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:13:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: Reconfiguring mds.foo.smithi152.adssru (monmap changed)... 2024-04-10T03:13:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:39 smithi152 ceph-mon[88640]: Reconfiguring daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: pgmap v133: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: Reconfiguring mds.foo.smithi152.adssru (monmap changed)... 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[98773]: Reconfiguring daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: pgmap v133: 89 pgs: 89 active+clean; 399 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: Reconfiguring mds.foo.smithi152.adssru (monmap changed)... 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.adssru", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:39 smithi102 ceph-mon[94535]: Reconfiguring daemon mds.foo.smithi152.adssru on smithi152 2024-04-10T03:13:40.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:40 smithi102 ceph-mon[98773]: pgmap v134: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:13:40.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:40 smithi102 ceph-mon[94535]: pgmap v134: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:13:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:40 smithi152 ceph-mon[88640]: pgmap v134: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-10T03:13:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:40] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2024-04-10T03:13:41.817 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Removed 132365 objects 2024-04-10T03:13:41.817 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Clean up completed and total clean up time :154.928 2024-04-10T03:13:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: pgmap v135: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: pgmap v136: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:42.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:42 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: pgmap v135: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: pgmap v136: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:13:42.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:13:42.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: pgmap v135: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-10T03:13:42.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: pgmap v136: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:42.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:42 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:13:44.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:44 smithi152 ceph-mon[88640]: pgmap v137: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:44.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:44 smithi102 ceph-mon[94535]: pgmap v137: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:44.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:44 smithi102 ceph-mon[98773]: pgmap v137: 89 pgs: 89 active+clean; 146 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-04-10T03:13:46.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:46 smithi152 ceph-mon[88640]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:46.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:46 smithi102 ceph-mon[94535]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:46.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:46 smithi102 ceph-mon[98773]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:46.988 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-04-10T03:13:46.988 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-04-10T03:13:47.254 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:13:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:48 smithi152 ceph-mon[88640]: pgmap v139: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:48 smithi102 ceph-mon[94535]: pgmap v139: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:48.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:48 smithi102 ceph-mon[98773]: pgmap v139: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-04-10T03:13:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:13:49.509 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_1' removed 2024-04-10T03:13:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1454895031' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:49 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1454895031' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:49 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:49.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1454895031' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:49.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:49 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:13:50.141 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-10T03:13:50.141 INFO:tasks.radosbench:Beginning radosbench... 2024-04-10T03:13:50.142 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-04-10T03:13:50.142 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool create unique_pool_2 16 2024-04-10T03:13:50.456 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:13:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:50 smithi152 ceph-mon[88640]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2024-04-10T03:13:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:50 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:13:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:50 smithi152 ceph-mon[88640]: osdmap e83: 8 total, 8 up, 8 in 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[94535]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[94535]: osdmap e83: 8 total, 8 up, 8 in 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[98773]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:13:50.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-mon[98773]: osdmap e83: 8 total, 8 up, 8 in 2024-04-10T03:13:51.107 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:13:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:13:50] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.43.0" 2024-04-10T03:13:52.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:52 smithi152 ceph-mon[88640]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:52.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:52 smithi102 ceph-mon[94535]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:52.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:52 smithi102 ceph-mon[98773]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:53.521 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_2' created 2024-04-10T03:13:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3552826299' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:53 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3552826299' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:53 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3552826299' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:53 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-10T03:13:54.226 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-04-10T03:13:54.484 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:13:54.518 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[94535]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:54.518 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-10T03:13:54.518 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[94535]: osdmap e84: 8 total, 8 up, 8 in 2024-04-10T03:13:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[98773]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-10T03:13:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:54 smithi102 ceph-mon[98773]: osdmap e84: 8 total, 8 up, 8 in 2024-04-10T03:13:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:54 smithi152 ceph-mon[88640]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:54 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-10T03:13:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:54 smithi152 ceph-mon[88640]: osdmap e84: 8 total, 8 up, 8 in 2024-04-10T03:13:55.615 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:55 smithi102 ceph-mon[98773]: osdmap e85: 8 total, 8 up, 8 in 2024-04-10T03:13:55.616 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:55 smithi102 ceph-mon[94535]: osdmap e85: 8 total, 8 up, 8 in 2024-04-10T03:13:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:55 smithi152 ceph-mon[88640]: osdmap e85: 8 total, 8 up, 8 in 2024-04-10T03:13:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[94535]: pgmap v146: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[94535]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:13:56.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[94535]: osdmap e86: 8 total, 8 up, 8 in 2024-04-10T03:13:56.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[98773]: pgmap v146: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:56.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[98773]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:13:56.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:56 smithi102 ceph-mon[98773]: osdmap e86: 8 total, 8 up, 8 in 2024-04-10T03:13:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:56 smithi152 ceph-mon[88640]: pgmap v146: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:56 smithi152 ceph-mon[88640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:13:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:56 smithi152 ceph-mon[88640]: osdmap e86: 8 total, 8 up, 8 in 2024-04-10T03:13:57.537 INFO:teuthology.orchestra.run.smithi102.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-04-10T03:13:57.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:57 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2800914809' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:57.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:57 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:57.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:57 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2800914809' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:57.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:57 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:57 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2800914809' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:57 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:13:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[94535]: pgmap v148: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:13:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[94535]: osdmap e87: 8 total, 8 up, 8 in 2024-04-10T03:13:58.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[98773]: pgmap v148: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:58.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:13:58.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:58 smithi102 ceph-mon[98773]: osdmap e87: 8 total, 8 up, 8 in 2024-04-10T03:13:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:58 smithi152 ceph-mon[88640]: pgmap v148: 89 pgs: 13 creating+peering, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:13:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:58 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:13:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:58 smithi152 ceph-mon[88640]: osdmap e87: 8 total, 8 up, 8 in 2024-04-10T03:13:59.203 DEBUG:teuthology.orchestra.run.smithi102:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-04-10T03:13:59.205 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-10T03:13:59.206 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-10T03:13:59.255 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:hints = 1 2024-04-10T03:13:59.255 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-10T03:13:59.255 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object prefix: benchmark_data_smithi102_105505 2024-04-10T03:13:59.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:13:59.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 0 0 0 0 0 0 - 0 2024-04-10T03:14:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:13:59 smithi152 ceph-mon[88640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:14:00.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:13:59 smithi102 ceph-mon[98773]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:14:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:13:59 smithi102 ceph-mon[94535]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:14:00.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 1 16 3950 3934 245.861 245.875 0.00364673 0.00405619 2024-04-10T03:14:00.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:00 smithi102 ceph-mon[94535]: pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:14:00.935 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:00 smithi102 ceph-mon[98773]: pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:14:01.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:00 smithi152 ceph-mon[88640]: pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:14:01.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 2 16 7527 7511 234.702 223.562 0.00473402 0.00425152 2024-04-10T03:14:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:00] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.43.0" 2024-04-10T03:14:02.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 3 15 10943 10928 227.652 213.562 0.00898742 0.0043834 2024-04-10T03:14:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:02 smithi152 ceph-mon[88640]: pgmap v151: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s wr, 49 op/s 2024-04-10T03:14:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:02 smithi102 ceph-mon[94535]: pgmap v151: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s wr, 49 op/s 2024-04-10T03:14:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:02 smithi102 ceph-mon[98773]: pgmap v151: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s wr, 49 op/s 2024-04-10T03:14:03.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 4 16 14133 14117 220.562 199.312 0.00392003 0.00452706 2024-04-10T03:14:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:04.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 5 16 17132 17116 213.934 187.438 0.00285341 0.00466782 2024-04-10T03:14:04.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:04 smithi152 ceph-mon[88640]: pgmap v152: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s 2024-04-10T03:14:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:04 smithi102 ceph-mon[94535]: pgmap v152: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s 2024-04-10T03:14:05.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:04 smithi102 ceph-mon[98773]: pgmap v152: 89 pgs: 89 active+clean; 22 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s 2024-04-10T03:14:05.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 6 15 19951 19936 207.651 176.25 0.00843365 0.00480993 2024-04-10T03:14:06.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 7 16 22616 22600 201.77 166.5 0.00507074 0.00495044 2024-04-10T03:14:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:06 smithi152 ceph-mon[88640]: pgmap v153: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-10T03:14:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:06 smithi102 ceph-mon[94535]: pgmap v153: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-10T03:14:07.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:06 smithi102 ceph-mon[98773]: pgmap v153: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-10T03:14:07.256 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 8 16 24613 24597 192.148 124.812 0.00545988 0.00519877 2024-04-10T03:14:08.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 9 15 26874 26859 186.506 141.375 0.00623407 0.00535593 2024-04-10T03:14:08.549 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade resume' 2024-04-10T03:14:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:08 smithi102 ceph-mon[94535]: pgmap v154: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-10T03:14:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:08 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:08.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:08 smithi102 ceph-mon[98773]: pgmap v154: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-10T03:14:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:08 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:08 smithi152 ceph-mon[88640]: pgmap v154: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-10T03:14:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:08 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:09.056 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:14:09.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 10 16 28836 28820 180.11 122.562 0.0074284 0.00554568 2024-04-10T03:14:10.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 11 16 30356 30340 172.372 95 0.00526028 0.0057955 2024-04-10T03:14:10.936 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:10 smithi102 ceph-mon[98773]: pgmap v155: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:14:10.937 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:10 smithi102 ceph-mon[94535]: pgmap v155: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:14:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:10 smithi152 ceph-mon[88640]: pgmap v155: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-04-10T03:14:11.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 12 16 32203 32187 167.627 115.438 0.00325354 0.00596004 2024-04-10T03:14:11.269 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:10] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2024-04-10T03:14:12.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 13 16 34303 34287 164.821 131.25 0.00487273 0.00606141 2024-04-10T03:14:12.721 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:12 smithi102 ceph-mon[94535]: pgmap v156: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-10T03:14:12.721 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:12 smithi102 ceph-mon[98773]: pgmap v156: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-10T03:14:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:12 smithi152 ceph-mon[88640]: pgmap v156: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-10T03:14:13.257 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 14 16 36201 36185 161.521 118.625 0.0165009 0.00618606 2024-04-10T03:14:13.266 INFO:teuthology.orchestra.run.smithi102.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:14.258 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 15 16 37635 37619 156.727 89.625 0.00451894 0.00637437 2024-04-10T03:14:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[94535]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:14:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[94535]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[94535]: pgmap v157: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-10T03:14:14.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[98773]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:14:14.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[98773]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:14.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:14.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:14.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:14 smithi102 ceph-mon[98773]: pgmap v157: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-10T03:14:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:14 smithi152 ceph-mon[88640]: from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:14:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:14 smithi152 ceph-mon[88640]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:14.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:14 smithi152 ceph-mon[88640]: pgmap v157: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-10T03:14:14.926 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'echo wait for half of osds to upgrade' 2024-04-10T03:14:15.258 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 16 16 39421 39405 153.905 111.625 0.00737372 0.00649204 2024-04-10T03:14:16.099 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:14:16.258 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 17 16 41210 41194 151.428 111.812 0.00847334 0.00659921 2024-04-10T03:14:16.504 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:16 smithi102 ceph-mon[94535]: pgmap v158: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 194 MiB/s wr, 3.10k op/s 2024-04-10T03:14:16.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:16 smithi152 ceph-mon[88640]: pgmap v158: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 194 MiB/s wr, 3.10k op/s 2024-04-10T03:14:16.764 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:16 smithi102 ceph-mon[98773]: pgmap v158: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 194 MiB/s wr, 3.10k op/s 2024-04-10T03:14:17.258 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 18 16 43012 42996 149.272 112.625 0.00698936 0.00669393 2024-04-10T03:14:18.258 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 19 16 44720 44704 147.034 106.75 0.00884264 0.00678956 2024-04-10T03:14:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:18 smithi152 ceph-mon[88640]: pgmap v159: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[94535]: pgmap v159: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[98773]: pgmap v159: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:18.825 INFO:teuthology.orchestra.run.smithi102.stdout:wait for half of osds to upgrade 2024-04-10T03:14:19.259 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:14:19.263386+0000 min lat: 0.000957011 max lat: 0.115398 avg lat: 0.00688081 2024-04-10T03:14:19.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:14:19.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 20 16 46493 46477 145.214 110.812 0.00662516 0.00688081 2024-04-10T03:14:19.697 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-10T03:14:20.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 21 16 48086 48070 143.039 99.5625 0.00717752 0.00698556 2024-04-10T03:14:20.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:20 smithi102 ceph-mon[94535]: pgmap v160: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-10T03:14:20.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:20 smithi102 ceph-mon[98773]: pgmap v160: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-10T03:14:20.933 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:14:21.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:20 smithi152 ceph-mon[88640]: pgmap v160: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-10T03:14:21.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 22 16 49694 49678 141.105 100.5 0.00783007 0.00708198 2024-04-10T03:14:21.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:20] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.43.0" 2024-04-10T03:14:22.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 23 16 51385 51369 139.565 105.688 0.0106241 0.00716017 2024-04-10T03:14:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:22 smithi102 ceph-mon[94535]: pgmap v161: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-10T03:14:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:22 smithi102 ceph-mon[98773]: pgmap v161: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-10T03:14:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:22 smithi152 ceph-mon[88640]: pgmap v161: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-10T03:14:23.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 24 16 52830 52814 137.513 90.3125 0.00627205 0.00726603 2024-04-10T03:14:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 25 16 54043 54027 135.045 75.8125 0.013609 0.0073975 2024-04-10T03:14:24.717 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-10T03:14:24.717 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.718 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3774301570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-10T03:14:24.719 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:24.720 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3774301570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: pgmap v162: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-10T03:14:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:25.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:25.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:25.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3774301570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:25.260 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 26 16 55301 55285 132.874 78.625 0.00807377 0.0075195 2024-04-10T03:14:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[98773]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[98773]: Upgrade: Setting container_image for all mon 2024-04-10T03:14:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[98773]: Upgrade: Setting container_image for all crash 2024-04-10T03:14:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[98773]: Upgrade: osd.0 is safe to restart 2024-04-10T03:14:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[94535]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[94535]: Upgrade: Setting container_image for all mon 2024-04-10T03:14:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[94535]: Upgrade: Setting container_image for all crash 2024-04-10T03:14:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:25 smithi102 ceph-mon[94535]: Upgrade: osd.0 is safe to restart 2024-04-10T03:14:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:25 smithi152 ceph-mon[88640]: pgmap v163: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-10T03:14:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:25 smithi152 ceph-mon[88640]: Upgrade: Setting container_image for all mon 2024-04-10T03:14:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:25 smithi152 ceph-mon[88640]: Upgrade: Setting container_image for all crash 2024-04-10T03:14:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:25 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-10T03:14:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:25 smithi152 ceph-mon[88640]: Upgrade: osd.0 is safe to restart 2024-04-10T03:14:26.261 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 27 16 56672 56656 131.125 85.6875 0.00697886 0.00762085 2024-04-10T03:14:27.261 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 28 16 57978 57962 129.357 81.625 0.014554 0.00772513 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: pgmap v164: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: Upgrade: Updating osd.0 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: Deploying daemon osd.0 on smithi102 2024-04-10T03:14:27.426 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2691543975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: pgmap v164: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: Upgrade: Updating osd.0 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: Deploying daemon osd.0 on smithi102 2024-04-10T03:14:27.427 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2691543975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: pgmap v164: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: Upgrade: Updating osd.0 2024-04-10T03:14:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-10T03:14:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:27.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: Deploying daemon osd.0 on smithi102 2024-04-10T03:14:27.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2691543975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:28.261 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 29 16 59518 59502 128.215 96.25 0.00617569 0.00779416 2024-04-10T03:14:29.261 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 30 16 60864 60848 126.745 84.125 0.00747716 0.00788398 2024-04-10T03:14:29.428 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:29 smithi102 ceph-mon[98773]: pgmap v165: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:29.428 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:29 smithi102 ceph-mon[94535]: pgmap v165: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:29.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:29 smithi152 ceph-mon[88640]: pgmap v165: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-10T03:14:30.261 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 31 16 62183 62167 125.316 82.4375 0.00869933 0.00797483 2024-04-10T03:14:30.467 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4029533917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:30.467 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4029533917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:30.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4029533917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:31.194 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:30] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.43.0" 2024-04-10T03:14:31.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 32 16 63586 63570 124.137 87.6875 0.00645283 0.00804998 2024-04-10T03:14:31.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:31 smithi152 ceph-mon[88640]: pgmap v166: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:31.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:31 smithi102 ceph-mon[98773]: pgmap v166: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:31.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:31 smithi102 ceph-mon[94535]: pgmap v166: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:32.041 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:31 smithi102 systemd[1]: Stopping Ceph osd.0 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:14:32.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 33 16 64767 64751 122.612 73.8125 0.0114665 0.0081508 2024-04-10T03:14:32.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:32 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/258780087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:32.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:32 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/258780087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:32.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:32 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/258780087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:33.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 34 16 65987 65971 121.248 76.25 0.0054248 0.00822565 2024-04-10T03:14:33.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:33 smithi152 ceph-mon[88640]: pgmap v167: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-10T03:14:33.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:33.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:33 smithi152 ceph-mon[88640]: osd.0 marked itself down and dead 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[94535]: pgmap v167: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[94535]: osd.0 marked itself down and dead 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[98773]: pgmap v167: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:33.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-mon[98773]: osd.0 marked itself down and dead 2024-04-10T03:14:33.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[43548]: 2024-04-10T03:14:33.154+0000 7fb710bfe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:14:33.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[43548]: 2024-04-10T03:14:33.154+0000 7fb710bfe700 -1 osd.0 87 *** Got signal Terminated *** 2024-04-10T03:14:33.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[43548]: 2024-04-10T03:14:33.154+0000 7fb710bfe700 -1 osd.0 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:14:34.092 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:33 smithi102 podman[106950]: 2024-04-10 03:14:33.815611167 +0000 UTC m=+1.733453634 container died 984949346b35cc2edc124f6e2e5eed59768cc8817509834dcaef06f98f6d8e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-d3b5060) 2024-04-10T03:14:34.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 35 16 66002 65986 117.811 0.9375 1.02864 0.00826504 2024-04-10T03:14:34.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:34 smithi152 ceph-mon[88640]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:14:34.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:34 smithi152 ceph-mon[88640]: osdmap e88: 8 total, 7 up, 8 in 2024-04-10T03:14:34.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:34 smithi102 ceph-mon[98773]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:14:34.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:34 smithi102 ceph-mon[98773]: osdmap e88: 8 total, 7 up, 8 in 2024-04-10T03:14:34.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:34 smithi102 ceph-mon[94535]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:14:34.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:34 smithi102 ceph-mon[94535]: osdmap e88: 8 total, 7 up, 8 in 2024-04-10T03:14:34.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:34 smithi102 podman[106950]: 2024-04-10 03:14:34.269459672 +0000 UTC m=+2.187302145 container cleanup 984949346b35cc2edc124f6e2e5eed59768cc8817509834dcaef06f98f6d8e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2) 2024-04-10T03:14:34.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:34 smithi102 bash[106950]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0 2024-04-10T03:14:35.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 36 16 67485 67469 117.113 92.6875 0.00930622 0.00853324 2024-04-10T03:14:35.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:35 smithi152 ceph-mon[88640]: pgmap v169: 89 pgs: 15 stale+active+clean, 74 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:35.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:35 smithi152 ceph-mon[88640]: osdmap e89: 8 total, 7 up, 8 in 2024-04-10T03:14:35.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3009974115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:35.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[98773]: pgmap v169: 89 pgs: 15 stale+active+clean, 74 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:35.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[98773]: osdmap e89: 8 total, 7 up, 8 in 2024-04-10T03:14:35.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3009974115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:35.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:35 smithi102 podman[106967]: 2024-04-10 03:14:35.21385668 +0000 UTC m=+1.390744064 container remove 984949346b35cc2edc124f6e2e5eed59768cc8817509834dcaef06f98f6d8e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2) 2024-04-10T03:14:35.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[94535]: pgmap v169: 89 pgs: 15 stale+active+clean, 74 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-10T03:14:35.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[94535]: osdmap e89: 8 total, 7 up, 8 in 2024-04-10T03:14:35.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3009974115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:36.041 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:35 smithi102 podman[107067]: 2024-04-10 03:14:35.627948878 +0000 UTC m=+0.033512898 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:36.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 37 16 69145 69129 116.752 103.75 0.00697129 0.00856071 2024-04-10T03:14:36.621 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:36 smithi102 ceph-mon[98773]: Health check failed: Reduced data availability: 8 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-10T03:14:36.621 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:36 smithi102 ceph-mon[98773]: Health check failed: Degraded data redundancy: 8217/198438 objects degraded (4.141%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:36.621 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:36 smithi102 ceph-mon[94535]: Health check failed: Reduced data availability: 8 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-10T03:14:36.621 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:36 smithi102 ceph-mon[94535]: Health check failed: Degraded data redundancy: 8217/198438 objects degraded (4.141%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:36.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:36 smithi152 ceph-mon[88640]: Health check failed: Reduced data availability: 8 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-10T03:14:36.763 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:36 smithi152 ceph-mon[88640]: Health check failed: Degraded data redundancy: 8217/198438 objects degraded (4.141%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:37.041 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:36 smithi102 podman[107067]: 2024-04-10 03:14:36.624434258 +0000 UTC m=+1.029998268 container create 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-10T03:14:37.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 38 16 70651 70635 116.156 94.125 0.0058104 0.0086043 2024-04-10T03:14:37.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:37 smithi152 ceph-mon[88640]: pgmap v171: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:37.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:37 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1559026885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:37.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:37 smithi102 ceph-mon[98773]: pgmap v171: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:37.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:37 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1559026885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:37.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:37 smithi102 ceph-mon[94535]: pgmap v171: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:37.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:37 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1559026885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:38.133 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:38 smithi102 podman[107067]: 2024-04-10 03:14:38.028260651 +0000 UTC m=+2.433824668 container init 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-10T03:14:38.134 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:38 smithi102 podman[107067]: 2024-04-10 03:14:38.035727922 +0000 UTC m=+2.441291928 container start 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:14:38.134 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:38 smithi102 podman[107067]: 2024-04-10 03:14:38.136596587 +0000 UTC m=+2.542160596 container attach 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-04-10T03:14:38.262 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 39 16 72108 72092 115.513 91.0625 0.00703174 0.00865261 2024-04-10T03:14:38.791 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:38 smithi102 podman[107117]: 2024-04-10 03:14:38.575533637 +0000 UTC m=+0.036506073 container died 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240403, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:14:39.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:14:39.266311+0000 min lat: 0.000957011 max lat: 1.15245 avg lat: 0.00867386 2024-04-10T03:14:39.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:14:39.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 40 16 73772 73756 115.225 104 0.01064 0.00867386 2024-04-10T03:14:39.679 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:39 smithi102 ceph-mon[98773]: pgmap v172: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:39.679 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:39 smithi102 ceph-mon[94535]: pgmap v172: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:39.679 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 podman[107117]: 2024-04-10 03:14:39.334616731 +0000 UTC m=+0.795589163 container remove 605c87bc78c72a207db193c12b51903666e7dbdcf56617e7c03c0abf92bc144c (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True) 2024-04-10T03:14:39.679 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.0.service: Deactivated successfully. 2024-04-10T03:14:39.679 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 systemd[1]: Stopped Ceph osd.0 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:14:39.679 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.0.service: Consumed 13min 50.115s CPU time. 2024-04-10T03:14:39.680 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 systemd[1]: Starting Ceph osd.0 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:14:39.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:39 smithi152 ceph-mon[88640]: pgmap v172: 89 pgs: 4 active+undersized+degraded, 25 peering, 5 active+undersized, 55 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 8217/198438 objects degraded (4.141%) 2024-04-10T03:14:40.041 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:39 smithi102 podman[107236]: 2024-04-10 03:14:39.906768317 +0000 UTC m=+0.032653064 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:40.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 41 16 75256 75240 114.676 92.75 0.00707226 0.00871561 2024-04-10T03:14:40.308 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:40 smithi102 podman[107236]: 2024-04-10 03:14:40.053100542 +0000 UTC m=+0.178985274 container create 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-10T03:14:40.308 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:40 smithi102 podman[107236]: 2024-04-10 03:14:40.311479853 +0000 UTC m=+0.437364595 container init 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-10T03:14:40.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:40 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1808897584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:40 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1808897584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:40.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:40 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1808897584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:40.792 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:40 smithi102 podman[107236]: 2024-04-10 03:14:40.31965432 +0000 UTC m=+0.445539057 container start 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:14:40.792 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:40 smithi102 podman[107236]: 2024-04-10 03:14:40.427612867 +0000 UTC m=+0.553497606 container attach 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403) 2024-04-10T03:14:41.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 42 16 76810 76794 114.258 97.125 0.0049129 0.00874762 2024-04-10T03:14:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:40] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2024-04-10T03:14:41.291 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-10T03:14:41.291 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 bash[107236]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate[107248]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 conmon[107248]: conmon 1211931b84319ce8b4c3 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5.scope/container/memory.events 2024-04-10T03:14:41.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 podman[107236]: 2024-04-10 03:14:41.136747705 +0000 UTC m=+1.262632442 container died 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-10T03:14:41.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:41 smithi152 ceph-mon[88640]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 36254/215004 objects degraded (16.862%); 32 KiB/s, 0 objects/s recovering 2024-04-10T03:14:41.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:41 smithi152 ceph-mon[88640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-04-10T03:14:41.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-mon[98773]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 36254/215004 objects degraded (16.862%); 32 KiB/s, 0 objects/s recovering 2024-04-10T03:14:41.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-mon[98773]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-04-10T03:14:41.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-mon[94535]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 36254/215004 objects degraded (16.862%); 32 KiB/s, 0 objects/s recovering 2024-04-10T03:14:41.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:41 smithi102 ceph-mon[94535]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 10 pgs peering) 2024-04-10T03:14:42.171 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:41 smithi102 podman[107369]: 2024-04-10 03:14:41.834113766 +0000 UTC m=+0.687948201 container remove 1211931b84319ce8b4c33ab12c01b0355f0666239453bc253074339e37d1b3f5 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-10T03:14:42.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 43 16 78357 78341 113.849 96.6875 0.0176608 0.00877842 2024-04-10T03:14:42.478 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:42 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3888749899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:42.479 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:42 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3888749899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:42.479 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 podman[107438]: 2024-04-10 03:14:42.07846947 +0000 UTC m=+0.034486696 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:14:42.479 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 podman[107438]: 2024-04-10 03:14:42.209478618 +0000 UTC m=+0.165495876 container create 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-10T03:14:42.479 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 podman[107438]: 2024-04-10 03:14:42.4793266 +0000 UTC m=+0.435343818 container init 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403) 2024-04-10T03:14:42.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:42 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3888749899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:42.791 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 podman[107438]: 2024-04-10 03:14:42.493923531 +0000 UTC m=+0.449940748 container start 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-10T03:14:42.791 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 bash[107438]: 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 2024-04-10T03:14:42.791 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:42 smithi102 systemd[1]: Started Ceph osd.0 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:14:43.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 44 16 79878 79862 113.422 95.0625 0.00504216 0.00881158 2024-04-10T03:14:43.348 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:14:43.050+0000 7f44efe247c0 -1 Falling back to public interface 2024-04-10T03:14:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:43 smithi152 ceph-mon[88640]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 36739/220938 objects degraded (16.629%); 30 KiB/s, 0 objects/s recovering 2024-04-10T03:14:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:43 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 36739/220938 objects degraded (16.629%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:43 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:43 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:43 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[94535]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 36739/220938 objects degraded (16.629%); 30 KiB/s, 0 objects/s recovering 2024-04-10T03:14:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 36739/220938 objects degraded (16.629%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[98773]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 36739/220938 objects degraded (16.629%); 30 KiB/s, 0 objects/s recovering 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 36739/220938 objects degraded (16.629%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:43.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:43 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:44.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 45 16 81356 81340 112.953 92.375 0.0070564 0.00884726 2024-04-10T03:14:45.263 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 46 16 82751 82735 112.393 87.1875 0.0208716 0.00889278 2024-04-10T03:14:45.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:45 smithi102 ceph-mon[94535]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 36739/220938 objects degraded (16.629%); 26 KiB/s, 0 objects/s recovering 2024-04-10T03:14:45.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:45 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1729814085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:45.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:45 smithi102 ceph-mon[98773]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 36739/220938 objects degraded (16.629%); 26 KiB/s, 0 objects/s recovering 2024-04-10T03:14:45.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:45 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1729814085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:45.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:45 smithi152 ceph-mon[88640]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 36739/220938 objects degraded (16.629%); 26 KiB/s, 0 objects/s recovering 2024-04-10T03:14:45.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:45 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1729814085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:46.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 47 16 84107 84091 111.804 84.75 0.0170824 0.00893907 2024-04-10T03:14:47.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 48 16 85417 85401 111.18 81.875 0.0214217 0.00898947 2024-04-10T03:14:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:47 smithi102 ceph-mon[94535]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 40600/243756 objects degraded (16.656%); 22 KiB/s, 0 objects/s recovering 2024-04-10T03:14:47.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:47 smithi102 ceph-mon[98773]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 40600/243756 objects degraded (16.656%); 22 KiB/s, 0 objects/s recovering 2024-04-10T03:14:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:47 smithi152 ceph-mon[88640]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 40600/243756 objects degraded (16.656%); 22 KiB/s, 0 objects/s recovering 2024-04-10T03:14:48.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 49 16 86674 86658 110.514 78.5625 0.0156893 0.00904312 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3892060974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 40600/243756 objects degraded (16.656%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3892060974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 40600/243756 objects degraded (16.656%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3892060974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 40600/243756 objects degraded (16.656%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:48.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:48.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:14:49.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 50 16 87831 87815 109.75 72.3125 0.00589998 0.00910611 2024-04-10T03:14:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:49 smithi152 ceph-mon[88640]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 40600/243756 objects degraded (16.656%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:49 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:49 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[94535]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 40600/243756 objects degraded (16.656%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[98773]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 40600/243756 objects degraded (16.656%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:49 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:50.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 51 16 88881 88865 108.885 65.625 0.0212168 0.00917883 2024-04-10T03:14:50.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:50 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/883544746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:50.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:50 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/883544746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:50 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/883544746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:51.264 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 52 16 90123 90107 108.284 77.625 0.0102465 0.0092301 2024-04-10T03:14:51.271 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:14:50] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2024-04-10T03:14:51.541 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:14:51.386+0000 7f44efe247c0 -1 osd.0 87 log_to_monitors true 2024-04-10T03:14:52.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:51 smithi152 ceph-mon[88640]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 43403/258543 objects degraded (16.788%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:52.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:51 smithi152 ceph-mon[88640]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:51 smithi152 ceph-mon[88640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[94535]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 43403/258543 objects degraded (16.788%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[94535]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[94535]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[98773]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 43403/258543 objects degraded (16.788%); 21 KiB/s, 0 objects/s recovering 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[98773]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:51 smithi102 ceph-mon[98773]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-10T03:14:52.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 53 16 91039 91023 107.321 57.25 0.0128324 0.00931249 2024-04-10T03:14:52.521 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:14:52.238+0000 7f44dd274700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: osdmap e90: 8 total, 7 up, 8 in 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: from='osd.0 ' entity='osd.0' 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 43403/258543 objects degraded (16.788%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/561942337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:52.783 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: osdmap e90: 8 total, 7 up, 8 in 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: from='osd.0 ' entity='osd.0' 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 43403/258543 objects degraded (16.788%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:52.784 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/561942337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: from='osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: osdmap e90: 8 total, 7 up, 8 in 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: from='osd.0 ' entity='osd.0' 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 43403/258543 objects degraded (16.788%), 15 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/561942337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:53.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 54 16 92322 92306 106.818 80.1875 0.00889932 0.00935347 2024-04-10T03:14:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:53 smithi152 ceph-mon[88640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:14:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:53 smithi152 ceph-mon[88640]: osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743] boot 2024-04-10T03:14:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:53 smithi152 ceph-mon[88640]: osdmap e91: 8 total, 8 up, 8 in 2024-04-10T03:14:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:14:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[94535]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:14:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[94535]: osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743] boot 2024-04-10T03:14:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[94535]: osdmap e91: 8 total, 8 up, 8 in 2024-04-10T03:14:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:14:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[98773]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:14:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[98773]: osd.0 [v2:172.21.15.102:6802/2888888743,v1:172.21.15.102:6803/2888888743] boot 2024-04-10T03:14:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[98773]: osdmap e91: 8 total, 8 up, 8 in 2024-04-10T03:14:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-10T03:14:54.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 55 16 92333 92317 104.889 0.6875 0.00383234 0.00935486 2024-04-10T03:14:55.247 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:14:55.069+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: osdmap e92: 8 total, 8 up, 8 in 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2650346316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:55.251 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.252 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.252 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.252 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:55.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 56 16 92333 92317 103.016 0 - 0.00935486 2024-04-10T03:14:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: osdmap e92: 8 total, 8 up, 8 in 2024-04-10T03:14:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2650346316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 43754/263298 objects degraded (16.618%) 2024-04-10T03:14:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: osdmap e92: 8 total, 8 up, 8 in 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2650346316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:14:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:14:55.543 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:56.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 57 16 93178 93162 102.135 26.4062 0.0203367 0.0097849 2024-04-10T03:14:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:56 smithi152 ceph-mon[88640]: pgmap v184: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:56 smithi152 ceph-mon[88640]: pgmap v185: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:56 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:56 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-10T03:14:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:56 smithi152 ceph-mon[88640]: osdmap e93: 8 total, 8 up, 8 in 2024-04-10T03:14:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[94535]: pgmap v184: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[94535]: pgmap v185: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-10T03:14:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[94535]: osdmap e93: 8 total, 8 up, 8 in 2024-04-10T03:14:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[98773]: pgmap v184: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[98773]: pgmap v185: 89 pgs: 3 remapped+peering, 12 peering, 4 active+undersized+degraded, 9 active+undersized, 61 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 10997/275202 objects degraded (3.996%) 2024-04-10T03:14:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:14:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-10T03:14:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:56 smithi102 ceph-mon[98773]: osdmap e93: 8 total, 8 up, 8 in 2024-04-10T03:14:57.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 58 16 94065 94049 101.33 55.4375 0.0210857 0.0098627 2024-04-10T03:14:58.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 59 16 94970 94954 100.571 56.5625 0.0248042 0.00993821 2024-04-10T03:14:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:58 smithi152 ceph-mon[88640]: pgmap v187: 89 pgs: 3 remapped+peering, 12 peering, 2 active+undersized+degraded, 72 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 10994/275202 objects degraded (3.995%) 2024-04-10T03:14:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:58 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3397065134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:14:58 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10994/275202 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[94535]: pgmap v187: 89 pgs: 3 remapped+peering, 12 peering, 2 active+undersized+degraded, 72 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 10994/275202 objects degraded (3.995%) 2024-04-10T03:14:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3397065134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10994/275202 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:58.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[98773]: pgmap v187: 89 pgs: 3 remapped+peering, 12 peering, 2 active+undersized+degraded, 72 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 10994/275202 objects degraded (3.995%) 2024-04-10T03:14:58.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3397065134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:14:58.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:14:58 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10994/275202 objects degraded (3.995%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:14:59.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:14:59.269036+0000 min lat: 0.000957011 max lat: 2.15449 avg lat: 0.0100074 2024-04-10T03:14:59.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:14:59.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 60 16 95903 95887 99.8667 58.3125 0.0182431 0.0100074 2024-04-10T03:15:00.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 61 16 96756 96740 99.1035 53.3125 0.0122918 0.0100843 2024-04-10T03:15:00.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:00 smithi102 ceph-mon[98773]: pgmap v188: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 9 peering, 2 active+undersized+degraded, 73 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 14194/276933 objects degraded (5.125%) 2024-04-10T03:15:00.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:00 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1855291045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:00.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:00 smithi102 ceph-mon[94535]: pgmap v188: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 9 peering, 2 active+undersized+degraded, 73 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 14194/276933 objects degraded (5.125%) 2024-04-10T03:15:00.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:00 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1855291045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:00 smithi152 ceph-mon[88640]: pgmap v188: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 9 peering, 2 active+undersized+degraded, 73 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 14194/276933 objects degraded (5.125%) 2024-04-10T03:15:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:00 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1855291045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:01.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 62 16 97675 97659 98.4314 57.4375 0.0195995 0.0101538 2024-04-10T03:15:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:00] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2024-04-10T03:15:02.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 63 16 98586 98570 97.7727 56.9375 0.0207075 0.0102217 2024-04-10T03:15:02.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:02 smithi102 ceph-mon[98773]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 12926/287757 objects degraded (4.492%); 1.4 MiB/s, 21 objects/s recovering 2024-04-10T03:15:02.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:02 smithi102 ceph-mon[94535]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 12926/287757 objects degraded (4.492%); 1.4 MiB/s, 21 objects/s recovering 2024-04-10T03:15:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:02 smithi152 ceph-mon[88640]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 12926/287757 objects degraded (4.492%); 1.4 MiB/s, 21 objects/s recovering 2024-04-10T03:15:03.265 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 64 16 99422 99406 97.0614 52.25 0.0219481 0.0102972 2024-04-10T03:15:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12926/287757 objects degraded (4.492%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/777574667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:15:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:03.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12926/287757 objects degraded (4.492%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:03.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/777574667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:03.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:15:03.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12926/287757 objects degraded (4.492%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:03 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/777574667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:15:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:04.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 65 16 100343 100327 96.4537 57.5625 0.0185083 0.0103623 2024-04-10T03:15:04.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:04 smithi102 ceph-mon[94535]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 12926/287757 objects degraded (4.492%); 1.1 MiB/s, 18 objects/s recovering 2024-04-10T03:15:04.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:04 smithi102 ceph-mon[98773]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 12926/287757 objects degraded (4.492%); 1.1 MiB/s, 18 objects/s recovering 2024-04-10T03:15:04.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:04 smithi152 ceph-mon[88640]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 12926/287757 objects degraded (4.492%); 1.1 MiB/s, 18 objects/s recovering 2024-04-10T03:15:05.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 66 16 101194 101178 95.7981 53.1875 0.0239839 0.0104325 2024-04-10T03:15:06.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 67 16 102081 102065 95.1956 55.4375 0.0166072 0.0104992 2024-04-10T03:15:06.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:06 smithi152 ceph-mon[88640]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 12751/301077 objects degraded (4.235%); 2.0 MiB/s, 32 objects/s recovering 2024-04-10T03:15:06.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:06 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2764538243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:06 smithi102 ceph-mon[94535]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 12751/301077 objects degraded (4.235%); 2.0 MiB/s, 32 objects/s recovering 2024-04-10T03:15:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:06 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2764538243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:06.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:06 smithi102 ceph-mon[98773]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 12751/301077 objects degraded (4.235%); 2.0 MiB/s, 32 objects/s recovering 2024-04-10T03:15:06.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:06 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2764538243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:07.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 68 16 102876 102860 94.5264 49.6875 0.0135426 0.0105742 2024-04-10T03:15:08.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 69 16 103578 103562 93.7923 43.875 0.0113638 0.0106564 2024-04-10T03:15:08.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[98773]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12751/301077 objects degraded (4.235%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:15:08.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12751/301077 objects degraded (4.235%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:08.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4204700855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:08.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[94535]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12751/301077 objects degraded (4.235%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:15:08.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12751/301077 objects degraded (4.235%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:08.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:08 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4204700855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:08.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:08 smithi152 ceph-mon[88640]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s; 12751/301077 objects degraded (4.235%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:15:08.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:08 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12751/301077 objects degraded (4.235%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:08.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:08 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4204700855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:09.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 70 16 104454 104438 93.2345 54.75 0.0141004 0.0107204 2024-04-10T03:15:10.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 71 16 105280 105264 92.6484 51.625 0.011411 0.0107875 2024-04-10T03:15:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-mon[98773]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12751/303483 objects degraded (4.202%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:15:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:10.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-mon[94535]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12751/303483 objects degraded (4.202%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:15:10.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:10.542 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:15:10.085+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:10 smithi152 ceph-mon[88640]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s; 12751/303483 objects degraded (4.202%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:15:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:11.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 72 16 106125 106109 92.0951 52.8125 0.0202176 0.010853 2024-04-10T03:15:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:10] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-10T03:15:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:11 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:11 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:11 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3486721546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3486721546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:11.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:11 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3486721546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:12.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 73 16 106997 106981 91.5801 54.5 0.02107 0.0109147 2024-04-10T03:15:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:12 smithi152 ceph-mon[88640]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12573/313467 objects degraded (4.011%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:12 smithi102 ceph-mon[98773]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12573/313467 objects degraded (4.011%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:12 smithi102 ceph-mon[94535]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 12573/313467 objects degraded (4.011%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:13.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 74 16 107797 107781 91.0181 50 0.0262601 0.0109817 2024-04-10T03:15:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:13 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12573/313467 objects degraded (4.011%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:13 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2253059930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:13.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:13 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12573/313467 objects degraded (4.011%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:13.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:13 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2253059930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:13.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:13 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12573/313467 objects degraded (4.011%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:13.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:13 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2253059930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:14.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 75 16 108644 108628 90.5104 52.9375 0.0195039 0.0110424 2024-04-10T03:15:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:14 smithi152 ceph-mon[88640]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 12573/313467 objects degraded (4.011%); 1.8 MiB/s, 29 objects/s recovering 2024-04-10T03:15:14.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:14 smithi102 ceph-mon[98773]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 12573/313467 objects degraded (4.011%); 1.8 MiB/s, 29 objects/s recovering 2024-04-10T03:15:14.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:14 smithi102 ceph-mon[94535]: pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 12573/313467 objects degraded (4.011%); 1.8 MiB/s, 29 objects/s recovering 2024-04-10T03:15:15.266 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 76 16 109491 109475 90.016 52.9375 0.0265599 0.0111039 2024-04-10T03:15:16.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 77 16 110294 110278 89.4987 50.1875 0.00969878 0.0111683 2024-04-10T03:15:16.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:16 smithi152 ceph-mon[88640]: pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12354/326028 objects degraded (3.789%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:15:16.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:16 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/845004808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:16.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:16 smithi102 ceph-mon[94535]: pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12354/326028 objects degraded (3.789%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:15:16.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:16 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/845004808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:16.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:16 smithi102 ceph-mon[98773]: pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12354/326028 objects degraded (3.789%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:15:16.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:16 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/845004808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:17.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 78 16 111153 111137 89.0395 53.6875 0.0119519 0.0112255 2024-04-10T03:15:18.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 79 16 111935 111919 88.5311 48.875 0.0246084 0.0112897 2024-04-10T03:15:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:18 smithi152 ceph-mon[88640]: pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 12354/326028 objects degraded (3.789%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12354/326028 objects degraded (3.789%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:18.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:18.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[98773]: pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 12354/326028 objects degraded (3.789%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:18.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12354/326028 objects degraded (3.789%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:18.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[94535]: pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 12354/326028 objects degraded (3.789%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12354/326028 objects degraded (3.789%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:18.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:19.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:15:19.270879+0000 min lat: 0.000957011 max lat: 2.15449 avg lat: 0.0113502 2024-04-10T03:15:19.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:15:19.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 80 16 112750 112734 88.0611 50.9375 0.0244053 0.0113502 2024-04-10T03:15:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/709907158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/709907158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/709907158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:20.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 81 16 113604 113588 87.6328 53.375 0.00888621 0.0114063 2024-04-10T03:15:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:20 smithi152 ceph-mon[88640]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 12354/328233 objects degraded (3.764%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:20.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:20 smithi102 ceph-mon[98773]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 12354/328233 objects degraded (3.764%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:20.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:20 smithi102 ceph-mon[94535]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 12354/328233 objects degraded (3.764%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:15:21.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 82 16 114371 114355 87.1487 47.9375 0.0112097 0.0114692 2024-04-10T03:15:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:20] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:15:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:21 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3921035208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:21.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:21 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3921035208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:21.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:21 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3921035208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:22.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 83 16 115196 115180 86.7199 51.5625 0.00732377 0.0115254 2024-04-10T03:15:22.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:22 smithi152 ceph-mon[88640]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 12155/338313 objects degraded (3.593%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:15:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:22 smithi102 ceph-mon[94535]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 12155/338313 objects degraded (3.593%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:15:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:22 smithi102 ceph-mon[98773]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 12155/338313 objects degraded (3.593%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:15:23.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 84 16 115979 115963 86.2701 48.9375 0.009429 0.0115851 2024-04-10T03:15:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:23 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12155/338313 objects degraded (3.593%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:23 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12155/338313 objects degraded (3.593%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:23 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12155/338313 objects degraded (3.593%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:24.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 85 16 116765 116749 85.8331 49.125 0.0157947 0.0116443 2024-04-10T03:15:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:24 smithi152 ceph-mon[88640]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 12155/338313 objects degraded (3.593%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/278916929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:24 smithi102 ceph-mon[94535]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 12155/338313 objects degraded (3.593%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/278916929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:24 smithi102 ceph-mon[98773]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 12155/338313 objects degraded (3.593%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/278916929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:25.267 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 86 16 117599 117583 85.4411 52.125 0.023521 0.0116989 2024-04-10T03:15:25.390 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:15:25.102+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:26.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 87 16 118353 118337 85.0006 47.125 0.0223867 0.0117591 2024-04-10T03:15:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:26 smithi152 ceph-mon[88640]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11935/350409 objects degraded (3.406%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:15:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:26 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:26 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3372480307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[94535]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11935/350409 objects degraded (3.406%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:15:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3372480307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:26.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[98773]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 11935/350409 objects degraded (3.406%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:15:26.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:26.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:26.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3372480307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:27.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 88 16 119159 119143 84.6071 50.375 0.0177796 0.0118137 2024-04-10T03:15:28.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 89 16 119945 119929 84.2084 49.125 0.017677 0.0118696 2024-04-10T03:15:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:28 smithi152 ceph-mon[88640]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 11935/350409 objects degraded (3.406%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:28 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11935/350409 objects degraded (3.406%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:28.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:28 smithi102 ceph-mon[98773]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 11935/350409 objects degraded (3.406%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:28.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:28 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11935/350409 objects degraded (3.406%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:28 smithi102 ceph-mon[94535]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 11935/350409 objects degraded (3.406%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:28.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:28 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11935/350409 objects degraded (3.406%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:29.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 90 16 120678 120662 83.7817 45.8125 0.00382352 0.0119264 2024-04-10T03:15:29.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:29 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4038961071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:29 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4038961071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:29.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:29 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4038961071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:30.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total time run: 90.053 2024-04-10T03:15:30.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total writes made: 120678 2024-04-10T03:15:30.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Write size: 65536 2024-04-10T03:15:30.268 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object size: 65536 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Bandwidth (MB/sec): 83.7548 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Bandwidth: 44.8413 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max bandwidth (MB/sec): 245.875 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min bandwidth (MB/sec): 0 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average IOPS: 1340 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev IOPS: 717.468 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max IOPS: 3934 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min IOPS: 0 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average Latency(s): 0.0119347 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Latency(s): 0.0283308 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max latency(s): 2.15449 2024-04-10T03:15:30.269 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min latency(s): 0.000957011 2024-04-10T03:15:30.275 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Cleaning up (deleting benchmark objects) 2024-04-10T03:15:30.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:30 smithi152 ceph-mon[88640]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 11935/352545 objects degraded (3.385%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:30.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:30 smithi102 ceph-mon[94535]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 11935/352545 objects degraded (3.385%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:30.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:30 smithi102 ceph-mon[98773]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 736 op/s; 11935/352545 objects degraded (3.385%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:15:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:30] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:15:31.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:31 smithi152 ceph-mon[88640]: osdmap e94: 8 total, 8 up, 8 in 2024-04-10T03:15:31.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:31 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2694931822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:31.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:31 smithi102 ceph-mon[94535]: osdmap e94: 8 total, 8 up, 8 in 2024-04-10T03:15:31.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:31 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2694931822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:31.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:31 smithi102 ceph-mon[98773]: osdmap e94: 8 total, 8 up, 8 in 2024-04-10T03:15:31.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:31 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2694931822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:32 smithi152 ceph-mon[88640]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 11685/361749 objects degraded (3.230%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:32 smithi152 ceph-mon[88640]: osdmap e95: 8 total, 8 up, 8 in 2024-04-10T03:15:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:32 smithi102 ceph-mon[94535]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 11685/361749 objects degraded (3.230%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:32 smithi102 ceph-mon[94535]: osdmap e95: 8 total, 8 up, 8 in 2024-04-10T03:15:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:32 smithi102 ceph-mon[98773]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 11685/361749 objects degraded (3.230%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:32 smithi102 ceph-mon[98773]: osdmap e95: 8 total, 8 up, 8 in 2024-04-10T03:15:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:33 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11685/361749 objects degraded (3.230%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:33 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11685/361749 objects degraded (3.230%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:33 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11685/361749 objects degraded (3.230%), 8 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:34 smithi152 ceph-mon[88640]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 472 op/s; 11685/361749 objects degraded (3.230%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:15:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:34 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/885247094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:34 smithi102 ceph-mon[98773]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 472 op/s; 11685/361749 objects degraded (3.230%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:15:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:34 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/885247094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:34 smithi102 ceph-mon[94535]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 472 op/s; 11685/361749 objects degraded (3.230%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:15:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/885247094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:36.843 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:36 smithi102 ceph-mon[98773]: pgmap v208: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 30 MiB/s wr, 826 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:36.844 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:36 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2449095362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:36.844 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:36 smithi102 ceph-mon[94535]: pgmap v208: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 30 MiB/s wr, 826 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:36.844 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:36 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2449095362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:36 smithi152 ceph-mon[88640]: pgmap v208: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 30 MiB/s wr, 826 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:36 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2449095362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:37 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11081/353961 objects degraded (3.131%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:38.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:37 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11081/353961 objects degraded (3.131%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:37 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11081/353961 objects degraded (3.131%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:38 smithi152 ceph-mon[88640]: pgmap v209: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 25 MiB/s wr, 737 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:39.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:38 smithi102 ceph-mon[98773]: pgmap v209: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 25 MiB/s wr, 737 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:38 smithi102 ceph-mon[94535]: pgmap v209: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 25 MiB/s wr, 737 op/s; 11081/353961 objects degraded (3.131%); 6.7 MiB/s, 106 objects/s recovering 2024-04-10T03:15:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:39 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3202991246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:39 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3202991246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:40.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:39 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3202991246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:40.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:15:40.119+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:40.932 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-mon[98773]: pgmap v210: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 120 B/s rd, 894 KiB/s wr, 426 op/s; 11081/351558 objects degraded (3.152%); 4.4 MiB/s, 71 objects/s recovering 2024-04-10T03:15:40.932 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:40.932 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-mon[94535]: pgmap v210: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 120 B/s rd, 894 KiB/s wr, 426 op/s; 11081/351558 objects degraded (3.152%); 4.4 MiB/s, 71 objects/s recovering 2024-04-10T03:15:40.932 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:40 smithi152 ceph-mon[88640]: pgmap v210: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 120 B/s rd, 894 KiB/s wr, 426 op/s; 11081/351558 objects degraded (3.152%); 4.4 MiB/s, 71 objects/s recovering 2024-04-10T03:15:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:40 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:40] "GET /metrics HTTP/1.1" 200 36287 "" "Prometheus/2.43.0" 2024-04-10T03:15:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:41 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:41 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:41 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1108883240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1108883240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:15:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:41 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1108883240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:42 smithi152 ceph-mon[88640]: pgmap v211: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 761 KiB/s wr, 694 op/s; 10834/341628 objects degraded (3.171%); 5.3 MiB/s, 85 objects/s recovering 2024-04-10T03:15:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:42 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10834/341628 objects degraded (3.171%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:42 smithi102 ceph-mon[94535]: pgmap v211: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 761 KiB/s wr, 694 op/s; 10834/341628 objects degraded (3.171%); 5.3 MiB/s, 85 objects/s recovering 2024-04-10T03:15:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:42 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10834/341628 objects degraded (3.171%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:42 smithi102 ceph-mon[98773]: pgmap v211: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 761 KiB/s wr, 694 op/s; 10834/341628 objects degraded (3.171%); 5.3 MiB/s, 85 objects/s recovering 2024-04-10T03:15:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:42 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10834/341628 objects degraded (3.171%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:44 smithi152 ceph-mon[88640]: pgmap v212: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 88 B/s rd, 661 KiB/s wr, 603 op/s; 10834/341628 objects degraded (3.171%); 4.6 MiB/s, 73 objects/s recovering 2024-04-10T03:15:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:44 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1518396709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:44 smithi102 ceph-mon[94535]: pgmap v212: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 88 B/s rd, 661 KiB/s wr, 603 op/s; 10834/341628 objects degraded (3.171%); 4.6 MiB/s, 73 objects/s recovering 2024-04-10T03:15:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:44 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1518396709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:45.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:44 smithi102 ceph-mon[98773]: pgmap v212: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 88 B/s rd, 661 KiB/s wr, 603 op/s; 10834/341628 objects degraded (3.171%); 4.6 MiB/s, 73 objects/s recovering 2024-04-10T03:15:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:44 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1518396709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:47.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:46 smithi152 ceph-mon[88640]: pgmap v213: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 634 KiB/s wr, 916 op/s; 10579/329469 objects degraded (3.211%); 5.8 MiB/s, 92 objects/s recovering 2024-04-10T03:15:47.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2705703409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:46 smithi102 ceph-mon[94535]: pgmap v213: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 634 KiB/s wr, 916 op/s; 10579/329469 objects degraded (3.211%); 5.8 MiB/s, 92 objects/s recovering 2024-04-10T03:15:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2705703409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:46 smithi102 ceph-mon[98773]: pgmap v213: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 634 KiB/s wr, 916 op/s; 10579/329469 objects degraded (3.211%); 5.8 MiB/s, 92 objects/s recovering 2024-04-10T03:15:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2705703409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:48.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:47 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10579/329469 objects degraded (3.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:48.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:47 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10579/329469 objects degraded (3.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:48.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:47 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10579/329469 objects degraded (3.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:48 smithi152 ceph-mon[88640]: pgmap v214: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 10579/329469 objects degraded (3.211%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:48 smithi102 ceph-mon[94535]: pgmap v214: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 10579/329469 objects degraded (3.211%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:48 smithi102 ceph-mon[98773]: pgmap v214: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 10579/329469 objects degraded (3.211%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:15:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1271687869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1271687869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1271687869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:50.932 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:50 smithi102 ceph-mon[94535]: pgmap v215: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 10579/327120 objects degraded (3.234%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:50.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:50 smithi102 ceph-mon[98773]: pgmap v215: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 10579/327120 objects degraded (3.234%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:50 smithi152 ceph-mon[88640]: pgmap v215: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 10579/327120 objects degraded (3.234%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:15:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:15:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:15:50] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2024-04-10T03:15:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:52 smithi152 ceph-mon[88640]: pgmap v216: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 10311/317451 objects degraded (3.248%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:15:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/842940554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:52 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10311/317451 objects degraded (3.248%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[98773]: pgmap v216: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 10311/317451 objects degraded (3.248%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:15:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/842940554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10311/317451 objects degraded (3.248%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:53.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[94535]: pgmap v216: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 10311/317451 objects degraded (3.248%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:15:53.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/842940554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:53.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:52 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10311/317451 objects degraded (3.248%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:54 smithi152 ceph-mon[88640]: pgmap v217: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10311/317451 objects degraded (3.248%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:15:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:54 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/749761636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:54 smithi102 ceph-mon[94535]: pgmap v217: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10311/317451 objects degraded (3.248%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:15:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:54 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/749761636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:54 smithi102 ceph-mon[98773]: pgmap v217: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 10311/317451 objects degraded (3.248%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:15:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:54 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/749761636' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:15:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:15:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:15:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:56 smithi152 ceph-mon[88640]: pgmap v218: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 10019/305562 objects degraded (3.279%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:15:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:56 smithi102 ceph-mon[94535]: pgmap v218: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 10019/305562 objects degraded (3.279%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:15:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:56 smithi102 ceph-mon[98773]: pgmap v218: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 10019/305562 objects degraded (3.279%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:15:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:57 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3517275635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:57 smithi152 ceph-mon[88640]: pgmap v219: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10019/305562 objects degraded (3.279%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:57 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10019/305562 objects degraded (3.279%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3517275635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[98773]: pgmap v219: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10019/305562 objects degraded (3.279%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10019/305562 objects degraded (3.279%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:15:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3517275635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:15:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[94535]: pgmap v219: 89 pgs: 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 10019/305562 objects degraded (3.279%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:15:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:57 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10019/305562 objects degraded (3.279%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:16:00.003 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[98773]: osdmap e96: 8 total, 8 up, 8 in 2024-04-10T03:16:00.003 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[98773]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9965/303612 objects degraded (3.282%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:00.003 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2103315671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:00.003 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[94535]: osdmap e96: 8 total, 8 up, 8 in 2024-04-10T03:16:00.003 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[94535]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9965/303612 objects degraded (3.282%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:00.004 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:15:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2103315671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:59 smithi152 ceph-mon[88640]: osdmap e96: 8 total, 8 up, 8 in 2024-04-10T03:16:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:59 smithi152 ceph-mon[88640]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 9965/303612 objects degraded (3.282%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:15:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2103315671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:01.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:00 smithi152 ceph-mon[88640]: osdmap e97: 8 total, 8 up, 8 in 2024-04-10T03:16:01.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:00 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:01.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:00 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:01.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:00] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2024-04-10T03:16:01.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[94535]: osdmap e97: 8 total, 8 up, 8 in 2024-04-10T03:16:01.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:01.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:01.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[98773]: osdmap e97: 8 total, 8 up, 8 in 2024-04-10T03:16:01.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:01.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:00 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:02.836 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[98773]: pgmap v223: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 9730/295614 objects degraded (3.291%); 4.5 MiB/s, 72 objects/s recovering 2024-04-10T03:16:02.836 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:02.836 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:02.836 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3251592633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:02.837 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[94535]: pgmap v223: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 9730/295614 objects degraded (3.291%); 4.5 MiB/s, 72 objects/s recovering 2024-04-10T03:16:02.837 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:02.837 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:02.837 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3251592633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:02 smithi152 ceph-mon[88640]: pgmap v223: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 9730/295614 objects degraded (3.291%); 4.5 MiB/s, 72 objects/s recovering 2024-04-10T03:16:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3251592633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:03.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:02 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:16:02.838+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 9730/295614 objects degraded (3.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:16:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:16:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: pgmap v224: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 9730/295614 objects degraded (3.291%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: pgmap v225: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9730/295614 objects degraded (3.291%); 3.2 MiB/s, 50 objects/s recovering 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 9730/295614 objects degraded (3.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: pgmap v224: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 9730/295614 objects degraded (3.291%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: pgmap v225: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9730/295614 objects degraded (3.291%); 3.2 MiB/s, 50 objects/s recovering 2024-04-10T03:16:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 9730/295614 objects degraded (3.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:16:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: pgmap v224: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 9730/295614 objects degraded (3.291%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: pgmap v225: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 9730/295614 objects degraded (3.291%); 3.2 MiB/s, 50 objects/s recovering 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:04.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:04.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:04 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:04.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:04 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:04.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3985503113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:05.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3985503113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3985503113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:06.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:05 smithi152 ceph-mon[88640]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 9387/289215 objects degraded (3.246%); 6.7 MiB/s, 107 objects/s recovering 2024-04-10T03:16:06.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:05 smithi102 ceph-mon[94535]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 9387/289215 objects degraded (3.246%); 6.7 MiB/s, 107 objects/s recovering 2024-04-10T03:16:06.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:05 smithi102 ceph-mon[98773]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 923 op/s; 9387/289215 objects degraded (3.246%); 6.7 MiB/s, 107 objects/s recovering 2024-04-10T03:16:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:07 smithi152 ceph-mon[88640]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9387/285672 objects degraded (3.286%); 4.7 MiB/s, 75 objects/s recovering 2024-04-10T03:16:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:07 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2727589241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:07 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 9387/285672 objects degraded (3.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[98773]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9387/285672 objects degraded (3.286%); 4.7 MiB/s, 75 objects/s recovering 2024-04-10T03:16:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2727589241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 9387/285672 objects degraded (3.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[94535]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 9387/285672 objects degraded (3.286%); 4.7 MiB/s, 75 objects/s recovering 2024-04-10T03:16:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2727589241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:07 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 9387/285672 objects degraded (3.286%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:09 smithi152 ceph-mon[88640]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 9387/285672 objects degraded (3.286%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:16:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:09 smithi102 ceph-mon[94535]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 9387/285672 objects degraded (3.286%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:16:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:09 smithi102 ceph-mon[98773]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 9387/285672 objects degraded (3.286%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:16:10.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:10 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1926930855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:10.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:10 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1926930855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:10 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1926930855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:10] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:16:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:11 smithi152 ceph-mon[88640]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9138/274563 objects degraded (3.328%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:11 smithi102 ceph-mon[94535]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9138/274563 objects degraded (3.328%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:12.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:11 smithi102 ceph-mon[98773]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9138/274563 objects degraded (3.328%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:16:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:12 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4082707198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:12 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 9138/274563 objects degraded (3.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:12 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4082707198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:12 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 9138/274563 objects degraded (3.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:12 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4082707198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:12 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 9138/274563 objects degraded (3.328%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:14.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:13 smithi152 ceph-mon[88640]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9138/274563 objects degraded (3.328%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:16:14.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:13 smithi102 ceph-mon[98773]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9138/274563 objects degraded (3.328%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:16:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:13 smithi102 ceph-mon[94535]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9138/274563 objects degraded (3.328%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:16:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:14 smithi152 ceph-mon[88640]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 8889/266823 objects degraded (3.331%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:16:15.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:14 smithi102 ceph-mon[98773]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 8889/266823 objects degraded (3.331%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:16:15.159 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:14 smithi102 ceph-mon[94535]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 8889/266823 objects degraded (3.331%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:16:16.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:15 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3070345707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:16.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3070345707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:16.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3070345707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:16 smithi152 ceph-mon[88640]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:16 smithi102 ceph-mon[94535]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:16 smithi102 ceph-mon[98773]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:18.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3356993749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:18.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:17 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8889/263151 objects degraded (3.378%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3356993749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:17 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8889/263151 objects degraded (3.378%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:18.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:17 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:16:17.859+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3356993749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:17 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8889/263151 objects degraded (3.378%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:18 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:18 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:18 smithi152 ceph-mon[88640]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[98773]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:16:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:18 smithi102 ceph-mon[94535]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 8889/263151 objects degraded (3.378%); 2.6 MiB/s, 41 objects/s recovering 2024-04-10T03:16:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3200336576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:20.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3200336576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:20.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3200336576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:21.194 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:20] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-10T03:16:21.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:21 smithi152 ceph-mon[88640]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 8626/251724 objects degraded (3.427%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:21.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:21 smithi102 ceph-mon[98773]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 8626/251724 objects degraded (3.427%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:21.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:21 smithi102 ceph-mon[94535]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 8626/251724 objects degraded (3.427%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:22 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8626/251724 objects degraded (3.427%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:22 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8626/251724 objects degraded (3.427%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:22 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8626/251724 objects degraded (3.427%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:23 smithi152 ceph-mon[88640]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 8626/251724 objects degraded (3.427%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:16:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:23 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/999711249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:23 smithi102 ceph-mon[98773]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 8626/251724 objects degraded (3.427%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:16:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:23 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/999711249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:23 smithi102 ceph-mon[94535]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 8626/251724 objects degraded (3.427%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:16:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:23 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/999711249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:25 smithi152 ceph-mon[88640]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8347/243990 objects degraded (3.421%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:16:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:25 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2436098994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:25 smithi102 ceph-mon[98773]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8347/243990 objects degraded (3.421%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:16:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:25 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2436098994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:25 smithi102 ceph-mon[94535]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8347/243990 objects degraded (3.421%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:16:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2436098994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:27 smithi152 ceph-mon[88640]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8347/240321 objects degraded (3.473%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:16:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:27 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8347/240321 objects degraded (3.473%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:27 smithi152 ceph-mon[88640]: osdmap e98: 8 total, 8 up, 8 in 2024-04-10T03:16:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[98773]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8347/240321 objects degraded (3.473%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:16:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8347/240321 objects degraded (3.473%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[98773]: osdmap e98: 8 total, 8 up, 8 in 2024-04-10T03:16:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[94535]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 8347/240321 objects degraded (3.473%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:16:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8347/240321 objects degraded (3.473%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:27 smithi102 ceph-mon[94535]: osdmap e98: 8 total, 8 up, 8 in 2024-04-10T03:16:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3543000250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:28 smithi152 ceph-mon[88640]: osdmap e99: 8 total, 8 up, 8 in 2024-04-10T03:16:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3543000250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:28 smithi102 ceph-mon[94535]: osdmap e99: 8 total, 8 up, 8 in 2024-04-10T03:16:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3543000250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:28 smithi102 ceph-mon[98773]: osdmap e99: 8 total, 8 up, 8 in 2024-04-10T03:16:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:29 smithi152 ceph-mon[88640]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 8347/240321 objects degraded (3.473%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:16:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:29 smithi102 ceph-mon[94535]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 8347/240321 objects degraded (3.473%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:16:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:29 smithi102 ceph-mon[98773]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 8347/240321 objects degraded (3.473%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:16:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2341372524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:31.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2341372524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:31.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2341372524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:31.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:30] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-10T03:16:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:31 smithi152 ceph-mon[88640]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7962/230841 objects degraded (3.449%); 5.2 MiB/s, 82 objects/s recovering 2024-04-10T03:16:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:31 smithi102 ceph-mon[94535]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7962/230841 objects degraded (3.449%); 5.2 MiB/s, 82 objects/s recovering 2024-04-10T03:16:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:31 smithi102 ceph-mon[98773]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7962/230841 objects degraded (3.449%); 5.2 MiB/s, 82 objects/s recovering 2024-04-10T03:16:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:32 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7962/230841 objects degraded (3.449%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:32 smithi152 ceph-mon[88640]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7962/230841 objects degraded (3.449%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:32 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7962/230841 objects degraded (3.449%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:32 smithi102 ceph-mon[94535]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7962/230841 objects degraded (3.449%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:33.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:16:32.885+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:32 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7962/230841 objects degraded (3.449%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:32 smithi102 ceph-mon[98773]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7962/230841 objects degraded (3.449%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:33 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:33 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1100259327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1100259327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1100259327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:34 smithi152 ceph-mon[88640]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7763/222963 objects degraded (3.482%); 4.6 MiB/s, 72 objects/s recovering 2024-04-10T03:16:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:34 smithi102 ceph-mon[98773]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7763/222963 objects degraded (3.482%); 4.6 MiB/s, 72 objects/s recovering 2024-04-10T03:16:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:34 smithi102 ceph-mon[94535]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7763/222963 objects degraded (3.482%); 4.6 MiB/s, 72 objects/s recovering 2024-04-10T03:16:36.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1589660535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:36.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1589660535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:36.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1589660535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:37.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:36 smithi152 ceph-mon[88640]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7763/219390 objects degraded (3.538%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:16:37.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:36 smithi102 ceph-mon[98773]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7763/219390 objects degraded (3.538%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:16:37.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:36 smithi102 ceph-mon[94535]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7763/219390 objects degraded (3.538%); 4.0 MiB/s, 64 objects/s recovering 2024-04-10T03:16:38.192 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:37 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7763/219390 objects degraded (3.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:38.193 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:37 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7763/219390 objects degraded (3.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:38.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:37 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7763/219390 objects degraded (3.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3213895501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:38 smithi102 ceph-mon[94535]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 7763/219390 objects degraded (3.538%); 3.6 MiB/s, 58 objects/s recovering 2024-04-10T03:16:39.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3213895501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:39.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:38 smithi102 ceph-mon[98773]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 7763/219390 objects degraded (3.538%); 3.6 MiB/s, 58 objects/s recovering 2024-04-10T03:16:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3213895501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:38 smithi152 ceph-mon[88640]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 7763/219390 objects degraded (3.538%); 3.6 MiB/s, 58 objects/s recovering 2024-04-10T03:16:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:40] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-10T03:16:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:41 smithi152 ceph-mon[88640]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 7581/208671 objects degraded (3.633%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:41 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/694985715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:41 smithi102 ceph-mon[94535]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 7581/208671 objects degraded (3.633%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:42.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:41 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/694985715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:41 smithi102 ceph-mon[98773]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 7581/208671 objects degraded (3.633%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:16:42.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:41 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/694985715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:42 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7581/208671 objects degraded (3.633%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:42 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7581/208671 objects degraded (3.633%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:42 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7581/208671 objects degraded (3.633%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:43 smithi152 ceph-mon[88640]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7581/208671 objects degraded (3.633%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:43 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2690694063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:44.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:43 smithi102 ceph-mon[94535]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7581/208671 objects degraded (3.633%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:44.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:43 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2690694063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:44.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:43 smithi102 ceph-mon[98773]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7581/208671 objects degraded (3.633%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:43 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2690694063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:46.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:45 smithi152 ceph-mon[88640]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7382/201240 objects degraded (3.668%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:45 smithi102 ceph-mon[94535]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7382/201240 objects degraded (3.668%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:45 smithi102 ceph-mon[98773]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 7382/201240 objects degraded (3.668%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:16:47.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2585081408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2585081408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2585081408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:48.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:47 smithi152 ceph-mon[88640]: pgmap v249: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:48.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:47 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7382/197712 objects degraded (3.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:48.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:47 smithi102 ceph-mon[94535]: pgmap v249: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:48.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:47 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7382/197712 objects degraded (3.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:48.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:16:47.911+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:48.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:47 smithi102 ceph-mon[98773]: pgmap v249: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:48.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:47 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7382/197712 objects degraded (3.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:48 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:48 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:48 smithi152 ceph-mon[88640]: pgmap v250: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[98773]: pgmap v250: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:16:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:16:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:16:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:48 smithi102 ceph-mon[94535]: pgmap v250: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 7382/197712 objects degraded (3.734%); 2.0 MiB/s, 31 objects/s recovering 2024-04-10T03:16:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2922879841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2922879841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2922879841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:50 smithi152 ceph-mon[88640]: pgmap v251: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 7174/186423 objects degraded (3.848%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:16:51.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:50 smithi102 ceph-mon[98773]: pgmap v251: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 7174/186423 objects degraded (3.848%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:16:51.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:16:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:16:50] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-10T03:16:51.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:50 smithi102 ceph-mon[94535]: pgmap v251: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 7174/186423 objects degraded (3.848%); 3.1 MiB/s, 49 objects/s recovering 2024-04-10T03:16:52.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1402934990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:52.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1402934990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:52.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1402934990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:52 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7174/186423 objects degraded (3.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:52 smithi152 ceph-mon[88640]: pgmap v252: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7174/186423 objects degraded (3.848%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:16:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:52 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7174/186423 objects degraded (3.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:52 smithi102 ceph-mon[94535]: pgmap v252: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7174/186423 objects degraded (3.848%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:16:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:52 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7174/186423 objects degraded (3.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:53.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:52 smithi102 ceph-mon[98773]: pgmap v252: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 7174/186423 objects degraded (3.848%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:16:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3150080065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3150080065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:54.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3150080065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:55 smithi152 ceph-mon[88640]: pgmap v253: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6950/178572 objects degraded (3.892%); 3.3 MiB/s, 52 objects/s recovering 2024-04-10T03:16:55.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:55 smithi102 ceph-mon[94535]: pgmap v253: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6950/178572 objects degraded (3.892%); 3.3 MiB/s, 52 objects/s recovering 2024-04-10T03:16:55.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:55 smithi102 ceph-mon[98773]: pgmap v253: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6950/178572 objects degraded (3.892%); 3.3 MiB/s, 52 objects/s recovering 2024-04-10T03:16:57.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/74157348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/74157348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/74157348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:16:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:57 smithi152 ceph-mon[88640]: pgmap v254: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:16:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:57 smithi102 ceph-mon[98773]: pgmap v254: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:16:58.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:57 smithi102 ceph-mon[94535]: pgmap v254: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:16:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:58 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6950/178572 objects degraded (3.892%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:59.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:58 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6950/178572 objects degraded (3.892%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:16:59.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:58 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6950/178572 objects degraded (3.892%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:59 smithi152 ceph-mon[88640]: pgmap v255: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:16:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3805907118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:59 smithi102 ceph-mon[94535]: pgmap v255: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:16:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3805907118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:59 smithi102 ceph-mon[98773]: pgmap v255: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6950/175311 objects degraded (3.964%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:16:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3805907118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:00] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-10T03:17:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:01 smithi152 ceph-mon[88640]: pgmap v256: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 6649/164301 objects degraded (4.047%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:17:02.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:01 smithi102 ceph-mon[98773]: pgmap v256: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 6649/164301 objects degraded (4.047%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:17:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:01 smithi102 ceph-mon[94535]: pgmap v256: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 6649/164301 objects degraded (4.047%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:17:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3723348100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:02 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6649/164301 objects degraded (4.047%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3723348100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:02 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6649/164301 objects degraded (4.047%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3723348100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:02 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6649/164301 objects degraded (4.047%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:03 smithi152 ceph-mon[88640]: pgmap v257: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 6649/164301 objects degraded (4.047%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:17:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[98773]: pgmap v257: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 6649/164301 objects degraded (4.047%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[94535]: pgmap v257: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 6649/164301 objects degraded (4.047%); 2.7 MiB/s, 43 objects/s recovering 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:04 smithi152 ceph-mon[88640]: osdmap e100: 8 total, 8 up, 8 in 2024-04-10T03:17:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3874367741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:04 smithi152 ceph-mon[88640]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6649/157575 objects degraded (4.220%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:17:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[94535]: osdmap e100: 8 total, 8 up, 8 in 2024-04-10T03:17:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3874367741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[94535]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6649/157575 objects degraded (4.220%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:17:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[98773]: osdmap e100: 8 total, 8 up, 8 in 2024-04-10T03:17:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3874367741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:04 smithi102 ceph-mon[98773]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6649/157575 objects degraded (4.220%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:17:05.877 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:05 smithi152 ceph-mon[88640]: osdmap e101: 8 total, 8 up, 8 in 2024-04-10T03:17:06.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:05 smithi102 ceph-mon[98773]: osdmap e101: 8 total, 8 up, 8 in 2024-04-10T03:17:06.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:05 smithi102 ceph-mon[94535]: osdmap e101: 8 total, 8 up, 8 in 2024-04-10T03:17:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:06 smithi152 ceph-mon[88640]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6491/154617 objects degraded (4.198%); 3.6 MiB/s, 57 objects/s recovering 2024-04-10T03:17:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:06 smithi102 ceph-mon[94535]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6491/154617 objects degraded (4.198%); 3.6 MiB/s, 57 objects/s recovering 2024-04-10T03:17:07.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:06 smithi102 ceph-mon[98773]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6491/154617 objects degraded (4.198%); 3.6 MiB/s, 57 objects/s recovering 2024-04-10T03:17:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:07 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3356749231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:07 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6491/154617 objects degraded (4.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3356749231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6491/154617 objects degraded (4.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3356749231' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:07 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6491/154617 objects degraded (4.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:09 smithi152 ceph-mon[88640]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 6273/153591 objects degraded (4.084%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[98773]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 6273/153591 objects degraded (4.084%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-mon[94535]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 6273/153591 objects degraded (4.084%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:10.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:09 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:17:09.544+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:17:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:17:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 6273/149103 objects degraded (4.207%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:17:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:10.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1122855465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 6273/149103 objects degraded (4.207%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:17:10.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1122855465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 6273/149103 objects degraded (4.207%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1122855465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:10.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:10.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:10 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:10] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:17:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:12 smithi152 ceph-mon[88640]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6134/144675 objects degraded (4.240%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:17:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:12 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2200315793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:12 smithi102 ceph-mon[94535]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6134/144675 objects degraded (4.240%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:17:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:12 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2200315793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:12 smithi102 ceph-mon[98773]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 6134/144675 objects degraded (4.240%); 4.2 MiB/s, 67 objects/s recovering 2024-04-10T03:17:12.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:12 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2200315793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:13 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6134/144675 objects degraded (4.240%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:13.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:13 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6134/144675 objects degraded (4.240%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:13.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:13 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6134/144675 objects degraded (4.240%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:14 smithi152 ceph-mon[88640]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6134/144009 objects degraded (4.259%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:17:14.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:14 smithi102 ceph-mon[94535]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6134/144009 objects degraded (4.259%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:17:14.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:14 smithi102 ceph-mon[98773]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6134/144009 objects degraded (4.259%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:17:15.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:15 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2299095733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:15.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2299095733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:15.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2299095733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:16 smithi152 ceph-mon[88640]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5989/133392 objects degraded (4.490%); 3.9 MiB/s, 62 objects/s recovering 2024-04-10T03:17:16.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:16 smithi102 ceph-mon[94535]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5989/133392 objects degraded (4.490%); 3.9 MiB/s, 62 objects/s recovering 2024-04-10T03:17:16.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:16 smithi102 ceph-mon[98773]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5989/133392 objects degraded (4.490%); 3.9 MiB/s, 62 objects/s recovering 2024-04-10T03:17:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3881487640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:17.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3881487640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:17.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3881487640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:18 smithi152 ceph-mon[88640]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5989/132666 objects degraded (4.514%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5989/132666 objects degraded (4.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[94535]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5989/132666 objects degraded (4.514%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5989/132666 objects degraded (4.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:18.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[98773]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5989/132666 objects degraded (4.514%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:17:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5989/132666 objects degraded (4.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:18.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:20 smithi152 ceph-mon[88640]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5989/128457 objects degraded (4.662%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:17:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2042160830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:20.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:20 smithi102 ceph-mon[94535]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5989/128457 objects degraded (4.662%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:17:20.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2042160830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:20.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:20 smithi102 ceph-mon[98773]: pgmap v268: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5989/128457 objects degraded (4.662%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:17:20.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2042160830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:20] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:17:22.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:22 smithi152 ceph-mon[88640]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5844/121971 objects degraded (4.791%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:22.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3047434219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:22 smithi102 ceph-mon[94535]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5844/121971 objects degraded (4.791%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:22.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3047434219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:22 smithi102 ceph-mon[98773]: pgmap v269: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5844/121971 objects degraded (4.791%); 2.2 MiB/s, 35 objects/s recovering 2024-04-10T03:17:22.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3047434219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:23 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5844/121971 objects degraded (4.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:23 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5844/121971 objects degraded (4.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:23 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5844/121971 objects degraded (4.791%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:24 smithi152 ceph-mon[88640]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5844/121227 objects degraded (4.821%); 1.5 MiB/s, 24 objects/s recovering 2024-04-10T03:17:24.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:24 smithi102 ceph-mon[98773]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5844/121227 objects degraded (4.821%); 1.5 MiB/s, 24 objects/s recovering 2024-04-10T03:17:24.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:24 smithi102 ceph-mon[94535]: pgmap v270: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5844/121227 objects degraded (4.821%); 1.5 MiB/s, 24 objects/s recovering 2024-04-10T03:17:24.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:17:24.561+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:25 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:25 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:25 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1263733771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1263733771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:25.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1263733771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:26 smithi152 ceph-mon[88640]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 5676/110478 objects degraded (5.138%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:17:26.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:26 smithi102 ceph-mon[94535]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 5676/110478 objects degraded (5.138%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:17:26.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:26 smithi102 ceph-mon[98773]: pgmap v271: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 5676/110478 objects degraded (5.138%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:17:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:28 smithi152 ceph-mon[88640]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5676/109743 objects degraded (5.172%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3110356731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:28 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5676/109743 objects degraded (5.172%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[94535]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5676/109743 objects degraded (5.172%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3110356731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5676/109743 objects degraded (5.172%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:28.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[98773]: pgmap v272: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5676/109743 objects degraded (5.172%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:28.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3110356731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:28.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:28 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5676/109743 objects degraded (5.172%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:30.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:30 smithi152 ceph-mon[88640]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 5676/105417 objects degraded (5.384%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:30.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3749927579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:30.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:30 smithi102 ceph-mon[94535]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 5676/105417 objects degraded (5.384%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:30.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3749927579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:30.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:30 smithi102 ceph-mon[98773]: pgmap v273: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 5676/105417 objects degraded (5.384%); 1.6 MiB/s, 26 objects/s recovering 2024-04-10T03:17:30.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3749927579' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:30] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-10T03:17:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:32 smithi152 ceph-mon[88640]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 5500/99111 objects degraded (5.549%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:17:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:32 smithi102 ceph-mon[94535]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 5500/99111 objects degraded (5.549%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:17:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:32 smithi102 ceph-mon[98773]: pgmap v274: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 5500/99111 objects degraded (5.549%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:17:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:33 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5500/99111 objects degraded (5.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3934780499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5500/99111 objects degraded (5.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3934780499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5500/99111 objects degraded (5.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3934780499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:33.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:34.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:34 smithi152 ceph-mon[88640]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5500/98331 objects degraded (5.593%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:34.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:34 smithi102 ceph-mon[94535]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5500/98331 objects degraded (5.593%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:34.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:34 smithi102 ceph-mon[98773]: pgmap v275: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5500/98331 objects degraded (5.593%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:35.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2308638844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:35.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2308638844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:35.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2308638844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:36.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:36 smithi152 ceph-mon[88640]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5328/87801 objects degraded (6.068%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:17:36.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:36 smithi102 ceph-mon[94535]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5328/87801 objects degraded (6.068%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:17:36.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:36 smithi102 ceph-mon[98773]: pgmap v276: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5328/87801 objects degraded (6.068%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:17:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[94535]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5328/87096 objects degraded (6.117%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5328/87096 objects degraded (6.117%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1651192733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[98773]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5328/87096 objects degraded (6.117%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5328/87096 objects degraded (6.117%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:38.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1651192733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:38 smithi152 ceph-mon[88640]: pgmap v277: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5328/87096 objects degraded (6.117%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:38 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5328/87096 objects degraded (6.117%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1651192733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:40.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:39 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:17:39.578+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[94535]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 5328/82950 objects degraded (6.423%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:40.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[98773]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 5328/82950 objects degraded (6.423%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:40.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:40 smithi152 ceph-mon[88640]: pgmap v278: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 5328/82950 objects degraded (6.423%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:17:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:40 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:40 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:40 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:40] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-10T03:17:41.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:41 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3774070125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:41.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:41 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3774070125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:42.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:41 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3774070125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:42.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:42 smithi102 ceph-mon[94535]: pgmap v279: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5135/76482 objects degraded (6.714%); 1.9 MiB/s, 45 objects/s recovering 2024-04-10T03:17:42.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:42 smithi102 ceph-mon[98773]: pgmap v279: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5135/76482 objects degraded (6.714%); 1.9 MiB/s, 45 objects/s recovering 2024-04-10T03:17:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:42 smithi152 ceph-mon[88640]: pgmap v279: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5135/76482 objects degraded (6.714%); 1.9 MiB/s, 45 objects/s recovering 2024-04-10T03:17:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:43 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5135/76482 objects degraded (6.714%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:43 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3937342803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:43.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:43 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5135/76482 objects degraded (6.714%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:43.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:43 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3937342803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:43 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5135/76482 objects degraded (6.714%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:43 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3937342803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:44 smithi152 ceph-mon[88640]: pgmap v280: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5135/75801 objects degraded (6.774%); 981 KiB/s, 30 objects/s recovering 2024-04-10T03:17:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:44 smithi102 ceph-mon[94535]: pgmap v280: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5135/75801 objects degraded (6.774%); 981 KiB/s, 30 objects/s recovering 2024-04-10T03:17:45.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:44 smithi102 ceph-mon[98773]: pgmap v280: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5135/75801 objects degraded (6.774%); 981 KiB/s, 30 objects/s recovering 2024-04-10T03:17:47.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:46 smithi152 ceph-mon[88640]: pgmap v281: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 4938/65082 objects degraded (7.587%); 981 KiB/s, 46 objects/s recovering 2024-04-10T03:17:47.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2058097796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:46 smithi102 ceph-mon[94535]: pgmap v281: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 4938/65082 objects degraded (7.587%); 981 KiB/s, 46 objects/s recovering 2024-04-10T03:17:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2058097796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:46 smithi102 ceph-mon[98773]: pgmap v281: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 4938/65082 objects degraded (7.587%); 981 KiB/s, 46 objects/s recovering 2024-04-10T03:17:47.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2058097796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:48 smithi152 ceph-mon[88640]: pgmap v282: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4938/64353 objects degraded (7.673%); 64 KiB/s, 32 objects/s recovering 2024-04-10T03:17:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4938/64353 objects degraded (7.673%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:48 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3952093757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[98773]: pgmap v282: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4938/64353 objects degraded (7.673%); 64 KiB/s, 32 objects/s recovering 2024-04-10T03:17:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4938/64353 objects degraded (7.673%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:49.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3952093757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[94535]: pgmap v282: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4938/64353 objects degraded (7.673%); 64 KiB/s, 32 objects/s recovering 2024-04-10T03:17:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4938/64353 objects degraded (7.673%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:17:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:48 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3952093757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:50.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:49 smithi152 ceph-mon[88640]: osdmap e102: 8 total, 8 up, 8 in 2024-04-10T03:17:50.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:49 smithi102 ceph-mon[98773]: osdmap e102: 8 total, 8 up, 8 in 2024-04-10T03:17:50.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:49 smithi102 ceph-mon[94535]: osdmap e102: 8 total, 8 up, 8 in 2024-04-10T03:17:50.932 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:50 smithi102 ceph-mon[94535]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4889/60003 objects degraded (8.148%); 77 KiB/s, 43 objects/s recovering 2024-04-10T03:17:50.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:50 smithi102 ceph-mon[94535]: osdmap e103: 8 total, 8 up, 8 in 2024-04-10T03:17:50.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:50 smithi102 ceph-mon[98773]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4889/60003 objects degraded (8.148%); 77 KiB/s, 43 objects/s recovering 2024-04-10T03:17:50.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:50 smithi102 ceph-mon[98773]: osdmap e103: 8 total, 8 up, 8 in 2024-04-10T03:17:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:50 smithi152 ceph-mon[88640]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 4889/60003 objects degraded (8.148%); 77 KiB/s, 43 objects/s recovering 2024-04-10T03:17:51.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:50 smithi152 ceph-mon[88640]: osdmap e103: 8 total, 8 up, 8 in 2024-04-10T03:17:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:17:50] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2024-04-10T03:17:52.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/276558036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:52.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/276558036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:52.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/276558036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:52 smithi152 ceph-mon[88640]: pgmap v286: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 4889/54978 objects degraded (8.893%); 0 B/s, 30 objects/s recovering 2024-04-10T03:17:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:52 smithi102 ceph-mon[98773]: pgmap v286: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 4889/54978 objects degraded (8.893%); 0 B/s, 30 objects/s recovering 2024-04-10T03:17:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:52 smithi102 ceph-mon[94535]: pgmap v286: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 4889/54978 objects degraded (8.893%); 0 B/s, 30 objects/s recovering 2024-04-10T03:17:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:53 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4889/54978 objects degraded (8.893%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2408150350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:53 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4889/54978 objects degraded (8.893%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:54.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2408150350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:53 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4889/54978 objects degraded (8.893%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:54.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2408150350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:55.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:54 smithi152 ceph-mon[88640]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 4768/53493 objects degraded (8.913%); 0 B/s, 21 objects/s recovering 2024-04-10T03:17:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:54 smithi102 ceph-mon[94535]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 4768/53493 objects degraded (8.913%); 0 B/s, 21 objects/s recovering 2024-04-10T03:17:55.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:17:54 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:17:54.600+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:54 smithi102 ceph-mon[98773]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 4768/53493 objects degraded (8.913%); 0 B/s, 21 objects/s recovering 2024-04-10T03:17:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:55 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:55 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:17:56.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:55 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:17:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:56 smithi152 ceph-mon[88640]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 917 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4495/44064 objects degraded (10.201%); 0 B/s, 55 objects/s recovering 2024-04-10T03:17:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1250574753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:56 smithi102 ceph-mon[94535]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 917 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4495/44064 objects degraded (10.201%); 0 B/s, 55 objects/s recovering 2024-04-10T03:17:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1250574753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:56 smithi102 ceph-mon[98773]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 917 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4495/44064 objects degraded (10.201%); 0 B/s, 55 objects/s recovering 2024-04-10T03:17:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1250574753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:17:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:57 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4495/44064 objects degraded (10.201%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:57 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4495/44064 objects degraded (10.201%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:58.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:57 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4495/44064 objects degraded (10.201%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:17:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:58 smithi152 ceph-mon[88640]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 887 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4495/42606 objects degraded (10.550%); 0 B/s, 50 objects/s recovering 2024-04-10T03:17:59.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:58 smithi102 ceph-mon[94535]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 887 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4495/42606 objects degraded (10.550%); 0 B/s, 50 objects/s recovering 2024-04-10T03:17:59.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:58 smithi102 ceph-mon[98773]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 887 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 4495/42606 objects degraded (10.550%); 0 B/s, 50 objects/s recovering 2024-04-10T03:18:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:17:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/799026089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:00.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:17:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/799026089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:17:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/799026089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:00.932 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:00 smithi102 ceph-mon[98773]: pgmap v290: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 795 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4298/38229 objects degraded (11.243%); 0 B/s, 59 objects/s recovering 2024-04-10T03:18:00.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:00 smithi102 ceph-mon[94535]: pgmap v290: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 795 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4298/38229 objects degraded (11.243%); 0 B/s, 59 objects/s recovering 2024-04-10T03:18:01.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:00 smithi152 ceph-mon[88640]: pgmap v290: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 795 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 4298/38229 objects degraded (11.243%); 0 B/s, 59 objects/s recovering 2024-04-10T03:18:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:00] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2024-04-10T03:18:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:01 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1276418118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:02.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:01 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1276418118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:01 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1276418118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:02 smithi152 ceph-mon[88640]: pgmap v291: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 677 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 4298/32550 objects degraded (13.204%); 0 B/s, 50 objects/s recovering 2024-04-10T03:18:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:02 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4298/32550 objects degraded (13.204%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:02 smithi102 ceph-mon[98773]: pgmap v291: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 677 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 4298/32550 objects degraded (13.204%); 0 B/s, 50 objects/s recovering 2024-04-10T03:18:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:02 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4298/32550 objects degraded (13.204%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:02 smithi102 ceph-mon[94535]: pgmap v291: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 677 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 4298/32550 objects degraded (13.204%); 0 B/s, 50 objects/s recovering 2024-04-10T03:18:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:02 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4298/32550 objects degraded (13.204%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:04 smithi152 ceph-mon[88640]: pgmap v292: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 647 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4298/31104 objects degraded (13.818%); 0 B/s, 49 objects/s recovering 2024-04-10T03:18:05.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1715805321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:05.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:04 smithi102 ceph-mon[98773]: pgmap v292: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 647 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4298/31104 objects degraded (13.818%); 0 B/s, 49 objects/s recovering 2024-04-10T03:18:05.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1715805321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:04 smithi102 ceph-mon[94535]: pgmap v292: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 647 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4298/31104 objects degraded (13.818%); 0 B/s, 49 objects/s recovering 2024-04-10T03:18:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1715805321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:06 smithi152 ceph-mon[88640]: pgmap v293: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 437 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 4100/21039 objects degraded (19.488%); 0 B/s, 55 objects/s recovering 2024-04-10T03:18:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:06 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/940971448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:06 smithi102 ceph-mon[94535]: pgmap v293: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 437 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 4100/21039 objects degraded (19.488%); 0 B/s, 55 objects/s recovering 2024-04-10T03:18:07.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:06 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/940971448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:07.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:06 smithi102 ceph-mon[98773]: pgmap v293: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 437 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 4100/21039 objects degraded (19.488%); 0 B/s, 55 objects/s recovering 2024-04-10T03:18:07.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:06 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/940971448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:07 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4100/21039 objects degraded (19.488%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:07 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4100/21039 objects degraded (19.488%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:07 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4100/21039 objects degraded (19.488%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:08 smithi152 ceph-mon[88640]: pgmap v294: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 408 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4100/19632 objects degraded (20.884%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:08 smithi102 ceph-mon[94535]: pgmap v294: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 408 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4100/19632 objects degraded (20.884%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:09.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:08 smithi102 ceph-mon[98773]: pgmap v294: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 408 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4100/19632 objects degraded (20.884%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3586136350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3586136350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3586136350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:10 smithi152 ceph-mon[88640]: pgmap v295: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3907/15432 objects degraded (25.318%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:11.017 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:10 smithi102 ceph-mon[94535]: pgmap v295: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3907/15432 objects degraded (25.318%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:11.017 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:10] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2024-04-10T03:18:11.017 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:10 smithi102 ceph-mon[98773]: pgmap v295: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3907/15432 objects degraded (25.318%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:11 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3662733894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:12.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:11 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3662733894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:11 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3662733894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:12 smithi152 ceph-mon[88640]: pgmap v296: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 209 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3907/10095 objects degraded (38.702%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:12 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3907/10095 objects degraded (38.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:12 smithi102 ceph-mon[98773]: pgmap v296: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 209 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3907/10095 objects degraded (38.702%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:12 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3907/10095 objects degraded (38.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:13.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:12 smithi102 ceph-mon[94535]: pgmap v296: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 209 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3907/10095 objects degraded (38.702%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:13.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:12 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3907/10095 objects degraded (38.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:14.197 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Removed 120678 objects 2024-04-10T03:18:14.197 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Clean up completed and total clean up time :163.919 2024-04-10T03:18:14.998 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[94535]: pgmap v297: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 183 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 3907/8814 objects degraded (44.327%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2033659835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[98773]: pgmap v297: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 183 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 3907/8814 objects degraded (44.327%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2033659835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:14.999 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:14 smithi152 ceph-mon[88640]: pgmap v297: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 183 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 3907/8814 objects degraded (44.327%); 0 B/s, 32 objects/s recovering 2024-04-10T03:18:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2033659835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:15.467 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-04-10T03:18:15.467 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-04-10T03:18:15.743 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:18:16.367+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: pgmap v298: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: pgmap v298: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:18:16.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: pgmap v298: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 48 objects/s recovering 2024-04-10T03:18:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:18:16.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:17.523 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: pgmap v299: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 36 objects/s recovering 2024-04-10T03:18:17.523 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: pgmap v300: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 45 objects/s recovering 2024-04-10T03:18:17.523 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: osdmap e104: 8 total, 8 up, 8 in 2024-04-10T03:18:17.523 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:17.523 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1103637826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: pgmap v299: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 36 objects/s recovering 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: pgmap v300: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 45 objects/s recovering 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: osdmap e104: 8 total, 8 up, 8 in 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:17.524 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:18:17.525 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1103637826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:17.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: pgmap v299: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 36 objects/s recovering 2024-04-10T03:18:17.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: pgmap v300: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 15 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3712/783 objects degraded (474.074%); 0 B/s, 45 objects/s recovering 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: osdmap e104: 8 total, 8 up, 8 in 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:18:17.594 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1103637826' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:18.320 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_2' removed 2024-04-10T03:18:18.652 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[94535]: osdmap e105: 8 total, 8 up, 8 in 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3712/783 objects degraded (474.074%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3485411348' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[98773]: osdmap e105: 8 total, 8 up, 8 in 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3712/783 objects degraded (474.074%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3485411348' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:18.653 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:18 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:18 smithi152 ceph-mon[88640]: osdmap e105: 8 total, 8 up, 8 in 2024-04-10T03:18:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3712/783 objects degraded (474.074%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:18:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:18 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3485411348' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:18 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:18:19.026 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-10T03:18:19.026 INFO:tasks.radosbench:Beginning radosbench... 2024-04-10T03:18:19.027 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-04-10T03:18:19.027 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool create unique_pool_3 16 2024-04-10T03:18:19.327 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:18:19.554 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[94535]: pgmap v303: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 3712/252 objects degraded (1473.016%); 0 B/s, 42 objects/s recovering 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[94535]: osdmap e106: 8 total, 8 up, 8 in 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3122932401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[98773]: pgmap v303: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 3712/252 objects degraded (1473.016%); 0 B/s, 42 objects/s recovering 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[98773]: osdmap e106: 8 total, 8 up, 8 in 2024-04-10T03:18:19.555 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3122932401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:19 smithi152 ceph-mon[88640]: pgmap v303: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 3712/252 objects degraded (1473.016%); 0 B/s, 42 objects/s recovering 2024-04-10T03:18:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:19 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:18:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:19 smithi152 ceph-mon[88640]: osdmap e106: 8 total, 8 up, 8 in 2024-04-10T03:18:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3122932401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:20.611 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:20 smithi102 ceph-mon[94535]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3712/252 objects degraded (1473.016%), 3 pgs degraded, 3 pgs undersized) 2024-04-10T03:18:20.612 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:20 smithi102 ceph-mon[98773]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3712/252 objects degraded (1473.016%), 3 pgs degraded, 3 pgs undersized) 2024-04-10T03:18:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:20 smithi152 ceph-mon[88640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3712/252 objects degraded (1473.016%), 3 pgs degraded, 3 pgs undersized) 2024-04-10T03:18:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:20] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-10T03:18:21.649 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_3' created 2024-04-10T03:18:21.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:21 smithi102 ceph-mon[94535]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:21.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:21 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2769738954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:21.944 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:21 smithi102 ceph-mon[98773]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:21.945 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:21 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2769738954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:21 smithi152 ceph-mon[88640]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:21 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2769738954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:22.388 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-04-10T03:18:22.642 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:18:22.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-10T03:18:22.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-10T03:18:22.771 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[94535]: osdmap e107: 8 total, 8 up, 8 in 2024-04-10T03:18:22.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-10T03:18:22.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-10T03:18:22.772 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:22 smithi102 ceph-mon[98773]: osdmap e107: 8 total, 8 up, 8 in 2024-04-10T03:18:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-10T03:18:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3988007114' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-10T03:18:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:22 smithi152 ceph-mon[88640]: osdmap e107: 8 total, 8 up, 8 in 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[94535]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[94535]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[94535]: osdmap e108: 8 total, 8 up, 8 in 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[98773]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[98773]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:18:23.990 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:23 smithi102 ceph-mon[98773]: osdmap e108: 8 total, 8 up, 8 in 2024-04-10T03:18:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:23 smithi152 ceph-mon[88640]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:23 smithi152 ceph-mon[88640]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-10T03:18:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:23 smithi152 ceph-mon[88640]: osdmap e108: 8 total, 8 up, 8 in 2024-04-10T03:18:24.666 INFO:teuthology.orchestra.run.smithi102.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-04-10T03:18:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1514152259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:24 smithi152 ceph-mon[88640]: osdmap e109: 8 total, 8 up, 8 in 2024-04-10T03:18:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/444439637' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:24 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1514152259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[94535]: osdmap e109: 8 total, 8 up, 8 in 2024-04-10T03:18:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/444439637' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1514152259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[98773]: osdmap e109: 8 total, 8 up, 8 in 2024-04-10T03:18:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/444439637' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:24 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-10T03:18:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[94535]: pgmap v310: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:18:25.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[94535]: osdmap e110: 8 total, 8 up, 8 in 2024-04-10T03:18:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[98773]: pgmap v310: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:25.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:18:25.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:25 smithi102 ceph-mon[98773]: osdmap e110: 8 total, 8 up, 8 in 2024-04-10T03:18:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:25 smithi152 ceph-mon[88640]: pgmap v310: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:25 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-10T03:18:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:25 smithi152 ceph-mon[88640]: osdmap e110: 8 total, 8 up, 8 in 2024-04-10T03:18:26.537 DEBUG:teuthology.orchestra.run.smithi102:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-04-10T03:18:26.540 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-10T03:18:26.540 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-10T03:18:26.585 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:hints = 1 2024-04-10T03:18:26.585 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-10T03:18:26.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object prefix: benchmark_data_smithi102_115744 2024-04-10T03:18:26.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:18:26.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 0 0 0 0 0 0 - 0 2024-04-10T03:18:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1425295602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1425295602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:27.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1425295602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:27.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 1 16 4146 4130 258.11 258.125 0.00779205 0.00386037 2024-04-10T03:18:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:27 smithi152 ceph-mon[88640]: pgmap v312: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:27 smithi152 ceph-mon[88640]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:18:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:27 smithi102 ceph-mon[94535]: pgmap v312: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:27 smithi102 ceph-mon[94535]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:18:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:27 smithi102 ceph-mon[98773]: pgmap v312: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-10T03:18:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:27 smithi102 ceph-mon[98773]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-10T03:18:28.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 2 16 7601 7585 237.016 215.938 0.00282054 0.0042094 2024-04-10T03:18:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3850584019' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3850584019' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3850584019' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:29.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 3 16 11157 11141 232.088 222.25 0.00260369 0.00430229 2024-04-10T03:18:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:29 smithi152 ceph-mon[88640]: pgmap v313: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s 2024-04-10T03:18:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:29 smithi102 ceph-mon[98773]: pgmap v313: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s 2024-04-10T03:18:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:29 smithi102 ceph-mon[94535]: pgmap v313: 89 pgs: 89 active+clean; 37 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s 2024-04-10T03:18:30.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 4 16 14324 14308 223.546 197.938 0.00620624 0.00446647 2024-04-10T03:18:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:30] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-10T03:18:31.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 5 16 17111 17095 213.671 174.188 0.00511568 0.00467338 2024-04-10T03:18:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:31 smithi152 ceph-mon[88640]: pgmap v314: 89 pgs: 89 active+clean; 413 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-04-10T03:18:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:31 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2652310762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[94535]: pgmap v314: 89 pgs: 89 active+clean; 413 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2652310762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[98773]: pgmap v314: 89 pgs: 89 active+clean; 413 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2652310762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:32.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 6 16 19767 19751 205.723 166 0.00453818 0.00485474 2024-04-10T03:18:32.961 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:32 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:32.962 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:32 smithi102 ceph-mon[98773]: Upgrade: osd.1 is safe to restart 2024-04-10T03:18:32.962 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:32 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:32.962 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:32 smithi102 ceph-mon[94535]: Upgrade: osd.1 is safe to restart 2024-04-10T03:18:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:32 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-10T03:18:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:32 smithi152 ceph-mon[88640]: Upgrade: osd.1 is safe to restart 2024-04-10T03:18:33.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 7 16 22189 22173 197.958 151.375 0.0527038 0.00501034 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[98773]: pgmap v315: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[94535]: pgmap v315: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:33.958 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:33 smithi152 ceph-mon[88640]: pgmap v315: 89 pgs: 89 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-10T03:18:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:34.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 8 16 24452 24436 190.891 141.438 0.00694857 0.00523221 2024-04-10T03:18:34.986 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2163124878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:34.986 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:34.986 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:18:34.986 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:34.987 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2163124878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:34.987 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:34.987 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:18:34.987 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:34 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:34 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2163124878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:34 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:34 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-10T03:18:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:34 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:18:35.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 9 16 26814 26798 186.082 147.625 0.00233354 0.00536905 2024-04-10T03:18:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:35 smithi152 ceph-mon[88640]: pgmap v316: 89 pgs: 89 active+clean; 763 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-10T03:18:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:35 smithi152 ceph-mon[88640]: Upgrade: Updating osd.1 2024-04-10T03:18:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:35 smithi152 ceph-mon[88640]: Deploying daemon osd.1 on smithi102 2024-04-10T03:18:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[98773]: pgmap v316: 89 pgs: 89 active+clean; 763 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-10T03:18:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[98773]: Upgrade: Updating osd.1 2024-04-10T03:18:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[98773]: Deploying daemon osd.1 on smithi102 2024-04-10T03:18:36.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[94535]: pgmap v316: 89 pgs: 89 active+clean; 763 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-10T03:18:36.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[94535]: Upgrade: Updating osd.1 2024-04-10T03:18:36.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:35 smithi102 ceph-mon[94535]: Deploying daemon osd.1 on smithi102 2024-04-10T03:18:36.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 10 16 29092 29076 181.71 142.375 0.00682307 0.00549745 2024-04-10T03:18:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:36 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1771446239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:37.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:36 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1771446239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:37.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:36 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1771446239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:37.586 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 11 16 31249 31233 177.446 134.812 0.006809 0.00563024 2024-04-10T03:18:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:37 smithi152 ceph-mon[88640]: pgmap v317: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-10T03:18:38.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:37 smithi102 ceph-mon[98773]: pgmap v317: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-10T03:18:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:37 smithi102 ceph-mon[94535]: pgmap v317: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-10T03:18:38.587 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 12 16 33192 33176 172.777 121.438 0.00768971 0.00578167 2024-04-10T03:18:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:38 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:38 smithi152 ceph-mon[88640]: pgmap v318: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-04-10T03:18:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2053475233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[94535]: pgmap v318: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2053475233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[98773]: pgmap v318: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-04-10T03:18:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2053475233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:39.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 13 16 34934 34918 167.847 108.875 0.0123276 0.00595218 2024-04-10T03:18:40.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 14 16 36329 36313 162.085 87.1875 0.00308069 0.00610925 2024-04-10T03:18:41.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:41 smithi102 ceph-mon[98773]: pgmap v319: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-10T03:18:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:40] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.43.0" 2024-04-10T03:18:41.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:41 smithi102 ceph-mon[94535]: pgmap v319: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-10T03:18:41.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:41 smithi152 ceph-mon[88640]: pgmap v319: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-04-10T03:18:41.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 15 16 37657 37641 156.813 83 0.139001 0.00633301 2024-04-10T03:18:42.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:42 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3236373316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:42.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:42 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3236373316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:42.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:42 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3236373316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:42.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 16 16 39334 39318 153.563 104.812 0.00312621 0.00650647 2024-04-10T03:18:43.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:43 smithi102 ceph-mon[98773]: pgmap v320: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-10T03:18:43.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:43 smithi102 ceph-mon[94535]: pgmap v320: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-10T03:18:43.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:43 smithi152 ceph-mon[88640]: pgmap v320: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-04-10T03:18:43.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 17 16 41016 41000 150.713 105.125 0.00716297 0.00662992 2024-04-10T03:18:44.444 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:44 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3426839251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:44.444 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:44 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3426839251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:44.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:44 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3426839251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:44.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 18 16 42616 42600 147.895 100 0.00380643 0.00675497 2024-04-10T03:18:45.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:45 smithi102 ceph-mon[94535]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:18:45.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:45 smithi102 ceph-mon[98773]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:18:45.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:45 smithi152 ceph-mon[88640]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-10T03:18:45.588 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 19 16 44546 44530 146.459 120.625 0.0158754 0.00682285 2024-04-10T03:18:46.294 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:46 smithi102 systemd[1]: Stopping Ceph osd.1 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:18:46.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:18:46.592548+0000 min lat: 0.000945777 max lat: 0.175681 avg lat: 0.00693565 2024-04-10T03:18:46.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:18:46.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 20 16 46126 46110 144.074 98.75 0.00486533 0.00693565 2024-04-10T03:18:46.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:46 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[49661]: 2024-04-10T03:18:46.502+0000 7f5b8e127700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:18:46.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:46 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[49661]: 2024-04-10T03:18:46.502+0000 7f5b8e127700 -1 osd.1 110 *** Got signal Terminated *** 2024-04-10T03:18:46.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:46 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[49661]: 2024-04-10T03:18:46.502+0000 7f5b8e127700 -1 osd.1 110 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:18:47.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:47 smithi152 ceph-mon[88640]: pgmap v322: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-04-10T03:18:47.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:47 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3081628307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:47.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:47 smithi152 ceph-mon[88640]: osd.1 marked itself down and dead 2024-04-10T03:18:47.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[98773]: pgmap v322: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-04-10T03:18:47.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3081628307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:47.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[98773]: osd.1 marked itself down and dead 2024-04-10T03:18:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[94535]: pgmap v322: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-04-10T03:18:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3081628307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:47.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:47 smithi102 ceph-mon[94535]: osd.1 marked itself down and dead 2024-04-10T03:18:47.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 21 16 47043 47027 139.942 57.3125 0.00176895 0.00696301 2024-04-10T03:18:47.955 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:47 smithi102 podman[116350]: 2024-04-10 03:18:47.667150669 +0000 UTC m=+1.321356006 container died 13648c191186c52fb878c5d3b4aaabed6bdc3d34e037885a8aec3ded6bdddee5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409) 2024-04-10T03:18:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[94535]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:18:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[94535]: osdmap e111: 8 total, 7 up, 8 in 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[94535]: osdmap e112: 8 total, 7 up, 8 in 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[98773]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[98773]: osdmap e111: 8 total, 7 up, 8 in 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:48 smithi102 ceph-mon[98773]: osdmap e112: 8 total, 7 up, 8 in 2024-04-10T03:18:48.293 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:48 smithi102 podman[116350]: 2024-04-10 03:18:48.133764636 +0000 UTC m=+1.787969989 container cleanup 13648c191186c52fb878c5d3b4aaabed6bdc3d34e037885a8aec3ded6bdddee5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-10T03:18:48.293 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:48 smithi102 bash[116350]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1 2024-04-10T03:18:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:48 smithi152 ceph-mon[88640]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:18:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:48 smithi152 ceph-mon[88640]: osdmap e111: 8 total, 7 up, 8 in 2024-04-10T03:18:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:18:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:48 smithi152 ceph-mon[88640]: osdmap e112: 8 total, 7 up, 8 in 2024-04-10T03:18:48.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 22 15 48122 48107 136.649 67.5 0.00749138 0.00731358 2024-04-10T03:18:48.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:48 smithi102 podman[116372]: 2024-04-10 03:18:48.494270044 +0000 UTC m=+0.821371737 container remove 13648c191186c52fb878c5d3b4aaabed6bdc3d34e037885a8aec3ded6bdddee5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, RELEASE=quincy-d3b5060, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-04-10T03:18:49.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[94535]: pgmap v324: 89 pgs: 4 peering, 12 stale+active+clean, 73 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-10T03:18:49.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[94535]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:18:49.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2216018058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:49.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[98773]: pgmap v324: 89 pgs: 4 peering, 12 stale+active+clean, 73 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-10T03:18:49.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[98773]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:18:49.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2216018058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:49.292 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:48 smithi102 podman[116466]: 2024-04-10 03:18:48.878938911 +0000 UTC m=+0.037972912 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:18:49.292 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 podman[116466]: 2024-04-10 03:18:49.012319149 +0000 UTC m=+0.171353155 container create 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-10T03:18:49.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:49 smithi152 ceph-mon[88640]: pgmap v324: 89 pgs: 4 peering, 12 stale+active+clean, 73 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-10T03:18:49.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:49 smithi152 ceph-mon[88640]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:18:49.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2216018058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:49.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 23 16 50018 50002 135.857 118.438 0.00262424 0.00734682 2024-04-10T03:18:49.729 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 podman[116466]: 2024-04-10 03:18:49.332441706 +0000 UTC m=+0.491475722 container init 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:18:49.729 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 podman[116466]: 2024-04-10 03:18:49.338407133 +0000 UTC m=+0.497441134 container start 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0) 2024-04-10T03:18:49.729 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 podman[116466]: 2024-04-10 03:18:49.732570559 +0000 UTC m=+0.891604558 container attach 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-10T03:18:50.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 conmon[116481]: conmon 46e4e96edfd7d7e02060 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9.scope/container/memory.events 2024-04-10T03:18:50.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:49 smithi102 podman[116494]: 2024-04-10 03:18:49.846328741 +0000 UTC m=+0.036658530 container died 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-10T03:18:50.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:50 smithi152 ceph-mon[88640]: Health check failed: Degraded data redundancy: 6164/141816 objects degraded (4.346%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:50.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:50 smithi102 ceph-mon[94535]: Health check failed: Degraded data redundancy: 6164/141816 objects degraded (4.346%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:50.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:50 smithi102 ceph-mon[98773]: Health check failed: Degraded data redundancy: 6164/141816 objects degraded (4.346%), 4 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:50.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 24 16 51471 51455 133.98 90.8125 0.00241287 0.0074546 2024-04-10T03:18:51.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:51 smithi102 ceph-mon[98773]: pgmap v326: 89 pgs: 4 active+undersized+degraded, 4 peering, 3 stale+active+clean, 20 active+undersized, 58 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 6164/141816 objects degraded (4.346%); 8.0 KiB/s, 0 objects/s recovering 2024-04-10T03:18:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:18:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:18:50] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2024-04-10T03:18:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:51 smithi102 ceph-mon[94535]: pgmap v326: 89 pgs: 4 active+undersized+degraded, 4 peering, 3 stale+active+clean, 20 active+undersized, 58 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 6164/141816 objects degraded (4.346%); 8.0 KiB/s, 0 objects/s recovering 2024-04-10T03:18:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:51 smithi152 ceph-mon[88640]: pgmap v326: 89 pgs: 4 active+undersized+degraded, 4 peering, 3 stale+active+clean, 20 active+undersized, 58 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 6164/141816 objects degraded (4.346%); 8.0 KiB/s, 0 objects/s recovering 2024-04-10T03:18:51.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 25 15 53264 53249 133.105 112.125 0.0071178 0.0075081 2024-04-10T03:18:51.762 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:51 smithi102 podman[116494]: 2024-04-10 03:18:51.373101164 +0000 UTC m=+1.563430951 container remove 46e4e96edfd7d7e0206094816403cc40d082e43af128b65671f0fd9ae0c327f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240403) 2024-04-10T03:18:51.762 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:51 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-04-10T03:18:51.762 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:51 smithi102 systemd[1]: Stopped Ceph osd.1 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:18:51.762 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:51 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.1.service: Consumed 17min 19.302s CPU time. 2024-04-10T03:18:51.762 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:51 smithi102 systemd[1]: Starting Ceph osd.1 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:18:52.391 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2081119526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:52.392 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2081119526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:52.392 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:52 smithi102 podman[116611]: 2024-04-10 03:18:51.972763974 +0000 UTC m=+0.033468322 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:18:52.393 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:52 smithi102 podman[116611]: 2024-04-10 03:18:52.099828342 +0000 UTC m=+0.160532686 container create b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-10T03:18:52.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2081119526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:52.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 26 16 55155 55139 132.529 118.125 0.00888968 0.00754065 2024-04-10T03:18:52.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:52 smithi102 podman[116611]: 2024-04-10 03:18:52.395042678 +0000 UTC m=+0.455747021 container init b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-10T03:18:52.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:52 smithi102 podman[116611]: 2024-04-10 03:18:52.400292429 +0000 UTC m=+0.460996769 container start b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:18:52.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:52 smithi102 podman[116611]: 2024-04-10 03:18:52.511818883 +0000 UTC m=+0.572523232 container attach b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-10T03:18:53.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:53 smithi152 ceph-mon[88640]: pgmap v327: 89 pgs: 8 active+undersized+degraded, 4 peering, 27 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 19005/147627 objects degraded (12.874%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:53.532 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-mon[98773]: pgmap v327: 89 pgs: 8 active+undersized+degraded, 4 peering, 27 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 19005/147627 objects degraded (12.874%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:53.532 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-mon[94535]: pgmap v327: 89 pgs: 8 active+undersized+degraded, 4 peering, 27 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 19005/147627 objects degraded (12.874%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:53.532 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-10T03:18:53.532 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-10T03:18:53.532 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-10T03:18:53.532 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 bash[116611]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate[116626]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-10T03:18:53.533 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 podman[116611]: 2024-04-10 03:18:53.193332389 +0000 UTC m=+1.254036734 container died b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:18:53.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 27 16 57067 57051 132.046 119.5 0.00910691 0.00756794 2024-04-10T03:18:54.262 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:54 smithi102 ceph-mon[94535]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-10T03:18:54.262 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:54 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3882488374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:54.262 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:54 smithi102 ceph-mon[98773]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-10T03:18:54.262 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:54 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3882488374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:54.263 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:53 smithi102 podman[116754]: 2024-04-10 03:18:53.951291349 +0000 UTC m=+0.749905487 container remove b7f4093fc4b820ac6e590d5157c1575ed3a9bc3f2924b60b51c7add8794dd1f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True) 2024-04-10T03:18:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:54 smithi152 ceph-mon[88640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-10T03:18:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:54 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3882488374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:54.541 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 podman[116830]: 2024-04-10 03:18:54.173072283 +0000 UTC m=+0.041748075 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:18:54.541 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 podman[116830]: 2024-04-10 03:18:54.327973247 +0000 UTC m=+0.196649023 container create 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-10T03:18:54.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 28 16 58866 58850 131.345 112.438 0.00597843 0.00760865 2024-04-10T03:18:55.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 podman[116830]: 2024-04-10 03:18:54.597523005 +0000 UTC m=+0.466198783 container init 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True) 2024-04-10T03:18:55.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 podman[116830]: 2024-04-10 03:18:54.605269929 +0000 UTC m=+0.473945702 container start 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:18:55.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 ceph-osd[116850]: -- 172.21.15.102:0/634868225 <== mon.0 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cee71db680 con 0x55cee71a5400 2024-04-10T03:18:55.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 bash[116830]: 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 2024-04-10T03:18:55.042 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:54 smithi102 systemd[1]: Started Ceph osd.1 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:18:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:55 smithi152 ceph-mon[88640]: pgmap v328: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 22435/153120 objects degraded (14.652%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[98773]: pgmap v328: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 22435/153120 objects degraded (14.652%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:55.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[94535]: pgmap v328: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 22435/153120 objects degraded (14.652%); 16 KiB/s, 0 objects/s recovering 2024-04-10T03:18:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:18:55.542 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:18:55 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:18:55.160+0000 7f10c4b957c0 -1 Falling back to public interface 2024-04-10T03:18:55.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 29 16 60501 60485 130.34 102.188 0.00930323 0.00766657 2024-04-10T03:18:56.589 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 30 16 62168 62152 129.467 104.188 0.00528829 0.00771877 2024-04-10T03:18:56.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.908 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.908 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.908 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:56.908 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:56 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:56 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:56 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:56 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:57.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:56 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:18:57.590 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 31 16 63855 63839 128.692 105.438 0.00856451 0.00776489 2024-04-10T03:18:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:57 smithi152 ceph-mon[88640]: pgmap v329: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 25727/173088 objects degraded (14.864%); 14 KiB/s, 0 objects/s recovering 2024-04-10T03:18:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:57 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 25727/173088 objects degraded (14.864%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:57 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/383562547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[98773]: pgmap v329: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 25727/173088 objects degraded (14.864%); 14 KiB/s, 0 objects/s recovering 2024-04-10T03:18:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 25727/173088 objects degraded (14.864%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/383562547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:58.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[94535]: pgmap v329: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 25727/173088 objects degraded (14.864%); 14 KiB/s, 0 objects/s recovering 2024-04-10T03:18:58.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 25727/173088 objects degraded (14.864%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:18:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:57 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/383562547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:58.590 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 32 16 65496 65480 127.875 102.562 0.0107851 0.0078144 2024-04-10T03:18:59.590 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 33 16 66991 66975 126.831 93.4375 0.0055686 0.00787941 2024-04-10T03:18:59.930 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:59 smithi102 ceph-mon[94535]: pgmap v330: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 26247/179646 objects degraded (14.610%); 13 KiB/s, 0 objects/s recovering 2024-04-10T03:18:59.930 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:18:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2812281570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:18:59.930 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:59 smithi102 ceph-mon[98773]: pgmap v330: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 26247/179646 objects degraded (14.610%); 13 KiB/s, 0 objects/s recovering 2024-04-10T03:18:59.930 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:18:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2812281570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:59 smithi152 ceph-mon[88640]: pgmap v330: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 26247/179646 objects degraded (14.610%); 13 KiB/s, 0 objects/s recovering 2024-04-10T03:19:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:18:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2812281570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:00.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 34 16 68564 68548 125.992 98.3125 0.0115341 0.00793219 2024-04-10T03:19:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:00] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2024-04-10T03:19:01.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 35 16 69991 69975 124.937 89.1875 0.0149589 0.00799893 2024-04-10T03:19:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:01 smithi152 ceph-mon[88640]: pgmap v331: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 27289/192258 objects degraded (14.194%); 11 KiB/s, 0 objects/s recovering 2024-04-10T03:19:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:01 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:01 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[94535]: pgmap v331: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 27289/192258 objects degraded (14.194%); 11 KiB/s, 0 objects/s recovering 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[98773]: pgmap v331: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s; 27289/192258 objects degraded (14.194%); 11 KiB/s, 0 objects/s recovering 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:01.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:01 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:02.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 36 16 71404 71388 123.92 88.3125 0.0102402 0.0080642 2024-04-10T03:19:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/567476484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:02.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/567476484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:02.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/567476484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:03.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 37 16 72777 72761 122.89 85.8125 0.00900125 0.00813221 2024-04-10T03:19:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: pgmap v332: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29269/198198 objects degraded (14.768%); 5.3 KiB/s, 0 objects/s recovering 2024-04-10T03:19:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 29269/198198 objects degraded (14.768%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:03.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: pgmap v332: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29269/198198 objects degraded (14.768%); 5.3 KiB/s, 0 objects/s recovering 2024-04-10T03:19:03.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 29269/198198 objects degraded (14.768%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:03.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: pgmap v332: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29269/198198 objects degraded (14.768%); 5.3 KiB/s, 0 objects/s recovering 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 29269/198198 objects degraded (14.768%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:03.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:03.788 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:19:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:19:03.663+0000 7f10c4b957c0 -1 osd.1 110 log_to_monitors true 2024-04-10T03:19:04.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 38 16 74357 74341 122.254 98.75 0.00501554 0.00817462 2024-04-10T03:19:04.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:04 smithi152 ceph-mon[88640]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T03:19:04.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:04 smithi102 ceph-mon[94535]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T03:19:04.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:04 smithi102 ceph-mon[98773]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-10T03:19:04.791 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:19:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:19:04.515+0000 7f10b5fed700 -1 osd.1 110 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:19:05.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 39 16 75820 75804 121.464 91.4375 0.00332552 0.00821175 2024-04-10T03:19:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:05 smithi152 ceph-mon[88640]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29744/203886 objects degraded (14.589%) 2024-04-10T03:19:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:05 smithi152 ceph-mon[88640]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T03:19:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:05 smithi152 ceph-mon[88640]: osdmap e113: 8 total, 7 up, 8 in 2024-04-10T03:19:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:05 smithi152 ceph-mon[88640]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:19:05.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:05 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1697869956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[98773]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29744/203886 objects degraded (14.589%) 2024-04-10T03:19:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[98773]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T03:19:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[98773]: osdmap e113: 8 total, 7 up, 8 in 2024-04-10T03:19:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[98773]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:19:05.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1697869956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[94535]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 29744/203886 objects degraded (14.589%) 2024-04-10T03:19:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[94535]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-10T03:19:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[94535]: osdmap e113: 8 total, 7 up, 8 in 2024-04-10T03:19:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[94535]: from='osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:19:05.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:05 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1697869956' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:06.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:19:06.595312+0000 min lat: 0.000945777 max lat: 1.0233 avg lat: 0.00821175 2024-04-10T03:19:06.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:19:06.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 40 16 75820 75804 118.427 0 - 0.00821175 2024-04-10T03:19:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:06 smithi152 ceph-mon[88640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:19:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:06 smithi152 ceph-mon[88640]: osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242] boot 2024-04-10T03:19:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:06 smithi152 ceph-mon[88640]: osdmap e114: 8 total, 8 up, 8 in 2024-04-10T03:19:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:06 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:19:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:06 smithi152 ceph-mon[88640]: osdmap e115: 8 total, 8 up, 8 in 2024-04-10T03:19:06.769 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[98773]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:19:06.769 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[98773]: osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242] boot 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[98773]: osdmap e114: 8 total, 8 up, 8 in 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[98773]: osdmap e115: 8 total, 8 up, 8 in 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[94535]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[94535]: osd.1 [v2:172.21.15.102:6810/3348168242,v1:172.21.15.102:6811/3348168242] boot 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[94535]: osdmap e114: 8 total, 8 up, 8 in 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-10T03:19:06.770 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:06 smithi102 ceph-mon[94535]: osdmap e115: 8 total, 8 up, 8 in 2024-04-10T03:19:07.508 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:19:07.301+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-10T03:19:07.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 41 16 75820 75804 115.539 0 - 0.00821175 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1158276722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:19:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:07.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:07 smithi152 ceph-mon[88640]: osdmap e116: 8 total, 8 up, 8 in 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1158276722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[98773]: osdmap e116: 8 total, 8 up, 8 in 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1158276722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:19:07.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:07.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:07 smithi102 ceph-mon[94535]: osdmap e116: 8 total, 8 up, 8 in 2024-04-10T03:19:08.591 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 42 16 76772 76756 114.205 19.8333 0.0160023 0.00875133 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[94535]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[94535]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 32485/220416 objects degraded (14.738%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[98773]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:08.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[98773]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-10T03:19:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:08 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 32485/220416 objects degraded (14.738%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:08 smithi152 ceph-mon[88640]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:08 smithi152 ceph-mon[88640]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 32485/220416 objects degraded (14.738%) 2024-04-10T03:19:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:08 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:08 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-10T03:19:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:08 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 32485/220416 objects degraded (14.738%), 10 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:09.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 43 16 77821 77805 113.073 65.5625 0.0096927 0.00883755 2024-04-10T03:19:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4127491486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:09.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4127491486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4127491486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:10.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 44 16 78930 78914 112.079 69.3125 0.0132814 0.00891578 2024-04-10T03:19:10.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:10 smithi102 ceph-mon[98773]: pgmap v341: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 25339/229008 objects degraded (11.065%) 2024-04-10T03:19:10.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:10 smithi102 ceph-mon[94535]: pgmap v341: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 25339/229008 objects degraded (11.065%) 2024-04-10T03:19:11.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:10 smithi152 ceph-mon[88640]: pgmap v341: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 25339/229008 objects degraded (11.065%) 2024-04-10T03:19:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:10] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2024-04-10T03:19:11.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 45 16 79903 79887 110.939 60.8125 0.0406237 0.00900822 2024-04-10T03:19:12.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 46 16 80762 80746 109.695 53.6875 0.0323898 0.00910944 2024-04-10T03:19:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:12 smithi102 ceph-mon[94535]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 10724/231903 objects degraded (4.624%); 12 KiB/s, 0 objects/s recovering 2024-04-10T03:19:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:12 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2551350590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:12 smithi102 ceph-mon[98773]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 10724/231903 objects degraded (4.624%); 12 KiB/s, 0 objects/s recovering 2024-04-10T03:19:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:12 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2551350590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:12 smithi152 ceph-mon[88640]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 10724/231903 objects degraded (4.624%); 12 KiB/s, 0 objects/s recovering 2024-04-10T03:19:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:12 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2551350590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:13.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 47 16 81779 81763 108.713 63.5625 0.0223307 0.00919285 2024-04-10T03:19:13.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:13 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10724/231903 objects degraded (4.624%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:13.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:13 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10724/231903 objects degraded (4.624%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:14.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:13 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10724/231903 objects degraded (4.624%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:14.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 48 16 82671 82655 107.61 55.75 0.0363772 0.00928699 2024-04-10T03:19:14.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:14 smithi102 ceph-mon[98773]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 12291/235554 objects degraded (5.218%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:19:14.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:14 smithi102 ceph-mon[94535]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 12291/235554 objects degraded (5.218%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:19:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:14 smithi152 ceph-mon[88640]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s; 12291/235554 objects degraded (5.218%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:19:15.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 49 16 83761 83745 106.804 68.125 0.00992424 0.00935738 2024-04-10T03:19:16.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:15 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1010621173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:16.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1010621173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:16.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1010621173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:16.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 50 16 84806 84790 105.974 65.3125 0.00769968 0.00943111 2024-04-10T03:19:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:16 smithi152 ceph-mon[88640]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 12291/242817 objects degraded (5.062%); 1.5 MiB/s, 23 objects/s recovering 2024-04-10T03:19:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:16 smithi102 ceph-mon[98773]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 12291/242817 objects degraded (5.062%); 1.5 MiB/s, 23 objects/s recovering 2024-04-10T03:19:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:16 smithi102 ceph-mon[94535]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 12291/242817 objects degraded (5.062%); 1.5 MiB/s, 23 objects/s recovering 2024-04-10T03:19:17.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 51 16 85786 85770 105.097 61.25 0.00541812 0.00951003 2024-04-10T03:19:18.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1433316637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:18.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1433316637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1433316637' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:18.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 52 16 86839 86823 104.341 65.8125 0.0218054 0.00957878 2024-04-10T03:19:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:18 smithi152 ceph-mon[88640]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12291/246558 objects degraded (4.985%); 1.2 MiB/s, 18 objects/s recovering 2024-04-10T03:19:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12291/246558 objects degraded (4.985%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:19.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[98773]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12291/246558 objects degraded (4.985%); 1.2 MiB/s, 18 objects/s recovering 2024-04-10T03:19:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12291/246558 objects degraded (4.985%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:19.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[94535]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 12291/246558 objects degraded (4.985%); 1.2 MiB/s, 18 objects/s recovering 2024-04-10T03:19:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12291/246558 objects degraded (4.985%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:19:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:19.592 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 53 16 87846 87830 103.56 62.9375 0.0104322 0.00965004 2024-04-10T03:19:20.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 54 16 88871 88855 102.828 64.0625 0.017675 0.00971956 2024-04-10T03:19:20.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:20 smithi102 ceph-mon[98773]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 12078/256977 objects degraded (4.700%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:19:20.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4052033433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:20.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:20 smithi102 ceph-mon[94535]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 12078/256977 objects degraded (4.700%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:19:20.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4052033433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:21.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:20 smithi152 ceph-mon[88640]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 12078/256977 objects degraded (4.700%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:19:21.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4052033433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:20] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2024-04-10T03:19:21.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 55 16 89895 89879 102.122 64 0.00468005 0.00978572 2024-04-10T03:19:22.588 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:19:22.318+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:22.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 56 16 90832 90816 101.344 58.5625 0.00879994 0.00986198 2024-04-10T03:19:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:22 smithi152 ceph-mon[88640]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 12078/261819 objects degraded (4.613%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:19:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:22 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1707765246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[98773]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 12078/261819 objects degraded (4.613%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1707765246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[94535]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 12078/261819 objects degraded (4.613%); 2.1 MiB/s, 33 objects/s recovering 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1707765246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:23.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 57 16 91858 91842 100.691 64.125 0.00986508 0.00992559 2024-04-10T03:19:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:23 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:23 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:23 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 12078/261819 objects degraded (4.613%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 12078/261819 objects degraded (4.613%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:23 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 12078/261819 objects degraded (4.613%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:24.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 58 16 92817 92801 99.9886 59.9375 0.0140116 0.00999481 2024-04-10T03:19:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:24 smithi152 ceph-mon[88640]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 11836/265626 objects degraded (4.456%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:19:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:24 smithi102 ceph-mon[98773]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 11836/265626 objects degraded (4.456%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:19:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:24 smithi102 ceph-mon[94535]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s; 11836/265626 objects degraded (4.456%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:19:25.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 59 16 93777 93761 99.3108 60 0.0094813 0.010064 2024-04-10T03:19:26.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:25 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2739097067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:26.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2739097067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:26.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:25 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2739097067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:26.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:19:26.597150+0000 min lat: 0.000945777 max lat: 2.26505 avg lat: 0.010123 2024-04-10T03:19:26.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:19:26.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 60 16 94805 94789 98.7264 64.25 0.00905161 0.010123 2024-04-10T03:19:27.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:26 smithi152 ceph-mon[88640]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11836/273039 objects degraded (4.335%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:27.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:26 smithi102 ceph-mon[98773]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11836/273039 objects degraded (4.335%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:27.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:26 smithi102 ceph-mon[94535]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 11836/273039 objects degraded (4.335%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:27.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 61 16 95746 95730 98.072 58.8125 0.0110298 0.0101907 2024-04-10T03:19:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1527385981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:28.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:27 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11836/273039 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1527385981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:27 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11836/273039 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1527385981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:28.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:27 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11836/273039 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:28.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 62 16 96709 96693 97.4609 60.1875 0.00923472 0.0102544 2024-04-10T03:19:29.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:28 smithi152 ceph-mon[88640]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 11836/276879 objects degraded (4.275%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:28 smithi102 ceph-mon[98773]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 11836/276879 objects degraded (4.275%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:28 smithi102 ceph-mon[94535]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 11836/276879 objects degraded (4.275%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:19:29.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 63 16 97703 97687 96.8999 62.125 0.00811237 0.0103149 2024-04-10T03:19:30.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 64 16 98596 98580 96.2579 55.8125 0.00853286 0.0103833 2024-04-10T03:19:30.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:30 smithi102 ceph-mon[98773]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11570/287106 objects degraded (4.030%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:19:30.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2115418169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:30.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:30 smithi102 ceph-mon[94535]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11570/287106 objects degraded (4.030%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:19:30.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2115418169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:30 smithi152 ceph-mon[88640]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 11570/287106 objects degraded (4.030%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:19:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2115418169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:30] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2024-04-10T03:19:31.593 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 65 16 99574 99558 95.7173 61.125 0.00868493 0.0104421 2024-04-10T03:19:32.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 66 16 100499 100483 95.1429 57.8125 0.0146011 0.0105046 2024-04-10T03:19:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:32 smithi152 ceph-mon[88640]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 11570/291435 objects degraded (3.970%); 2.6 MiB/s, 42 objects/s recovering 2024-04-10T03:19:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:32 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11570/291435 objects degraded (3.970%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:32 smithi102 ceph-mon[98773]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 11570/291435 objects degraded (3.970%); 2.6 MiB/s, 42 objects/s recovering 2024-04-10T03:19:33.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:32 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11570/291435 objects degraded (3.970%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:32 smithi102 ceph-mon[94535]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 11570/291435 objects degraded (3.970%); 2.6 MiB/s, 42 objects/s recovering 2024-04-10T03:19:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:32 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11570/291435 objects degraded (3.970%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:33.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 67 15 101442 101427 94.6034 59 0.00983799 0.0105647 2024-04-10T03:19:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1884651604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1884651604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:34.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1884651604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:34.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:34.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 68 16 102406 102390 94.0972 60.1875 0.0107012 0.0106216 2024-04-10T03:19:34.911 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:34 smithi152 ceph-mon[88640]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 11291/294954 objects degraded (3.828%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:19:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:34 smithi102 ceph-mon[94535]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 11291/294954 objects degraded (3.828%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:19:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:34 smithi102 ceph-mon[98773]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 11291/294954 objects degraded (3.828%); 4.1 MiB/s, 65 objects/s recovering 2024-04-10T03:19:35.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 69 16 103303 103287 93.5459 56.0625 0.00862395 0.0106834 2024-04-10T03:19:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1539491970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:36.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1539491970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:36.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1539491970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:36.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 70 16 104265 104249 93.0684 60.125 0.0310789 0.010739 2024-04-10T03:19:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:36 smithi152 ceph-mon[88640]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 11291/302025 objects degraded (3.738%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:37.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:36 smithi102 ceph-mon[98773]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 11291/302025 objects degraded (3.738%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:37.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:36 smithi102 ceph-mon[94535]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 11291/302025 objects degraded (3.738%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:37.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 71 16 105175 105159 92.5586 56.875 0.0238373 0.0107986 2024-04-10T03:19:37.685 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:37 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:19:37.334+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:37 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:37 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 11291/302025 objects degraded (3.738%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:37 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:37 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 11291/302025 objects degraded (3.738%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:38.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:37 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:38.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:37 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 11291/302025 objects degraded (3.738%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:38.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 72 16 106067 106051 92.0473 55.75 0.00976978 0.0108584 2024-04-10T03:19:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:38 smithi152 ceph-mon[88640]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 11291/305574 objects degraded (3.695%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:38 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:38 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2000086159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[94535]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 11291/305574 objects degraded (3.695%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:39.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2000086159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[98773]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 11291/305574 objects degraded (3.695%); 2.8 MiB/s, 45 objects/s recovering 2024-04-10T03:19:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-10T03:19:39.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2000086159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:39.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 73 16 107016 107000 91.5988 59.3125 0.0141708 0.0109116 2024-04-10T03:19:40.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 74 16 107879 107863 91.0898 53.9375 0.0255953 0.0109724 2024-04-10T03:19:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:40 smithi152 ceph-mon[88640]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10988/315159 objects degraded (3.486%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:19:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:40 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3775879706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:41.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:40 smithi102 ceph-mon[94535]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10988/315159 objects degraded (3.486%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:19:41.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:40 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3775879706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:41.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:40] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-10T03:19:41.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:40 smithi102 ceph-mon[98773]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 10988/315159 objects degraded (3.486%); 4.4 MiB/s, 70 objects/s recovering 2024-04-10T03:19:41.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:40 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3775879706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:41.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 75 16 108779 108763 90.6253 56.25 0.015853 0.0110286 2024-04-10T03:19:42.594 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 76 16 109703 109687 90.1926 57.75 0.0127801 0.011082 2024-04-10T03:19:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:42 smithi152 ceph-mon[88640]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 10988/319401 objects degraded (3.440%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:19:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:42 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10988/319401 objects degraded (3.440%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:42 smithi152 ceph-mon[88640]: osdmap e117: 8 total, 8 up, 8 in 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[94535]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 10988/319401 objects degraded (3.440%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10988/319401 objects degraded (3.440%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[94535]: osdmap e117: 8 total, 8 up, 8 in 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[98773]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 10988/319401 objects degraded (3.440%); 3.0 MiB/s, 48 objects/s recovering 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10988/319401 objects degraded (3.440%), 7 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:43.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:42 smithi102 ceph-mon[98773]: osdmap e117: 8 total, 8 up, 8 in 2024-04-10T03:19:43.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 77 16 110067 110051 89.3167 22.75 0.00463329 0.0110825 2024-04-10T03:19:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:43 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3673310042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:44.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:43 smithi152 ceph-mon[88640]: osdmap e118: 8 total, 8 up, 8 in 2024-04-10T03:19:44.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:43 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3673310042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:44.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:43 smithi102 ceph-mon[98773]: osdmap e118: 8 total, 8 up, 8 in 2024-04-10T03:19:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:43 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3673310042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:43 smithi102 ceph-mon[94535]: osdmap e118: 8 total, 8 up, 8 in 2024-04-10T03:19:44.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 78 16 110932 110916 88.8647 54.0625 0.0262455 0.0112476 2024-04-10T03:19:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:44 smithi152 ceph-mon[88640]: pgmap v359: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10724/322869 objects degraded (3.321%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:19:45.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:44 smithi102 ceph-mon[98773]: pgmap v359: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10724/322869 objects degraded (3.321%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:19:45.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:44 smithi102 ceph-mon[94535]: pgmap v359: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s; 10724/322869 objects degraded (3.321%); 3.5 MiB/s, 56 objects/s recovering 2024-04-10T03:19:45.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 79 16 111822 111806 88.4439 55.625 0.00490018 0.011301 2024-04-10T03:19:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:45 smithi102 ceph-mon[98773]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10508/328890 objects degraded (3.195%); 6.1 MiB/s, 97 objects/s recovering 2024-04-10T03:19:46.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:45 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/782374668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:45 smithi102 ceph-mon[94535]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10508/328890 objects degraded (3.195%); 6.1 MiB/s, 97 objects/s recovering 2024-04-10T03:19:46.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:45 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/782374668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:46.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:45 smithi152 ceph-mon[88640]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10508/328890 objects degraded (3.195%); 6.1 MiB/s, 97 objects/s recovering 2024-04-10T03:19:46.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:45 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/782374668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:46.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:2024-04-10T03:19:46.598953+0000 min lat: 0.000945777 max lat: 2.26505 avg lat: 0.0113519 2024-04-10T03:19:46.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-10T03:19:46.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 80 16 112729 112713 88.0469 56.6875 0.0487014 0.0113519 2024-04-10T03:19:47.598 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 81 15 113656 113641 87.6759 58 0.00675053 0.0114002 2024-04-10T03:19:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:48 smithi152 ceph-mon[88640]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 10508/331899 objects degraded (3.166%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:19:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10508/331899 objects degraded (3.166%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:48.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[98773]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 10508/331899 objects degraded (3.166%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:19:48.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10508/331899 objects degraded (3.166%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:48.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:48.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[94535]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 10508/331899 objects degraded (3.166%); 3.7 MiB/s, 59 objects/s recovering 2024-04-10T03:19:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10508/331899 objects degraded (3.166%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:48.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:19:48.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 82 16 114472 114456 87.2278 50.9375 0.0271319 0.0114586 2024-04-10T03:19:49.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2148143158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:49.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2148143158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:49.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 83 16 115412 115396 86.8846 58.75 0.0305965 0.011504 2024-04-10T03:19:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2148143158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:50.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:50 smithi102 ceph-mon[98773]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 10260/340215 objects degraded (3.016%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:19:50.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:50 smithi102 ceph-mon[94535]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 10260/340215 objects degraded (3.016%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:19:50.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 84 16 116286 116270 86.5006 54.625 0.0286207 0.0115548 2024-04-10T03:19:50.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:50 smithi152 ceph-mon[88640]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s; 10260/340215 objects degraded (3.016%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:19:51.289 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:19:50] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-10T03:19:51.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/757460998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:51.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/757460998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:51.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 85 16 117164 117148 86.1284 54.875 0.0255883 0.0116054 2024-04-10T03:19:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/757460998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:52.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:52 smithi102 ceph-mon[98773]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 10260/345207 objects degraded (2.972%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:19:52.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:19:52 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:19:52.351+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:19:52.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:52 smithi102 ceph-mon[94535]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 10260/345207 objects degraded (2.972%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:19:52.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 86 16 118072 118056 85.7868 56.75 0.0209376 0.0116511 2024-04-10T03:19:52.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:52 smithi152 ceph-mon[88640]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 10260/345207 objects degraded (2.972%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:19:53.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 87 16 118905 118889 85.3991 52.0625 0.010676 0.0117043 2024-04-10T03:19:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:53 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:53 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:19:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:53 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10260/345207 objects degraded (2.972%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10260/345207 objects degraded (2.972%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:19:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:19:53.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:53 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10260/345207 objects degraded (2.972%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:54.595 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 88 16 119790 119774 85.0572 55.3125 0.0140769 0.0117506 2024-04-10T03:19:54.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:54 smithi152 ceph-mon[88640]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10260/347703 objects degraded (2.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:19:54.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:54 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2334211031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:54 smithi102 ceph-mon[98773]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10260/347703 objects degraded (2.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:19:54.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:54 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2334211031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:54.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:54 smithi102 ceph-mon[94535]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10260/347703 objects degraded (2.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-10T03:19:54.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:54 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2334211031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:55.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 89 16 120655 120639 84.7089 54.0625 0.0439113 0.0117999 2024-04-10T03:19:56.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout: 90 8 121478 121470 84.3447 51.9375 0.0213479 0.0118506 2024-04-10T03:19:56.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:56 smithi152 ceph-mon[88640]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10017/355320 objects degraded (2.819%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:19:56.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/437924555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:56.790 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:56 smithi102 ceph-mon[94535]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10017/355320 objects degraded (2.819%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:19:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/437924555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:56.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:56 smithi102 ceph-mon[98773]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10017/355320 objects degraded (2.819%); 3.8 MiB/s, 61 objects/s recovering 2024-04-10T03:19:56.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/437924555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total time run: 90.0182 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Total writes made: 121478 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Write size: 65536 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Object size: 65536 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Bandwidth (MB/sec): 84.3426 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Bandwidth: 44.7181 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max bandwidth (MB/sec): 258.125 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min bandwidth (MB/sec): 0 2024-04-10T03:19:57.596 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average IOPS: 1349 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev IOPS: 715.495 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max IOPS: 4130 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min IOPS: 0 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Average Latency(s): 0.0118512 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Stddev Latency(s): 0.0310925 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Max latency(s): 2.26505 2024-04-10T03:19:57.597 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Min latency(s): 0.000945777 2024-04-10T03:19:57.603 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Cleaning up (deleting benchmark objects) 2024-04-10T03:19:58.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:58 smithi152 ceph-mon[88640]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10017/358539 objects degraded (2.794%); 2.6 MiB/s, 40 objects/s recovering 2024-04-10T03:19:58.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:58 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 10017/358539 objects degraded (2.794%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:58 smithi102 ceph-mon[98773]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10017/358539 objects degraded (2.794%); 2.6 MiB/s, 40 objects/s recovering 2024-04-10T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:58 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 10017/358539 objects degraded (2.794%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:58 smithi102 ceph-mon[94535]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10017/358539 objects degraded (2.794%); 2.6 MiB/s, 40 objects/s recovering 2024-04-10T03:19:58.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:58 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 10017/358539 objects degraded (2.794%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:19:59.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:19:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3623068807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:59.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:19:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3623068807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:19:59.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:19:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3623068807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 915 op/s; 9535/361608 objects degraded (2.637%); 5.1 MiB/s, 81 objects/s recovering 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: [WRN] PG_DEGRADED: Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.0 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.3 is active+recovering+undersized+degraded+remapped, acting [5,6] 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-04-10T03:20:00.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-10T03:20:00.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:00 smithi152 ceph-mon[88640]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 915 op/s; 9535/361608 objects degraded (2.637%); 5.1 MiB/s, 81 objects/s recovering 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: [WRN] PG_DEGRADED: Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.0 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.3 is active+recovering+undersized+degraded+remapped, acting [5,6] 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-04-10T03:20:00.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[94535]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 54 MiB/s wr, 915 op/s; 9535/361608 objects degraded (2.637%); 5.1 MiB/s, 81 objects/s recovering 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: daemon grafana.a on smithi152 is in error state 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: [WRN] PG_DEGRADED: Degraded data redundancy: 9535/361608 objects degraded (2.637%), 6 pgs degraded 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.0 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.3 is active+recovering+undersized+degraded+remapped, acting [5,6] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.a is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.b is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-10T03:20:00.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-mon[98773]: pg 8.f is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2024-04-10T03:20:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:00] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-10T03:20:01.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:01 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4240879384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:01.790 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:01 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4240879384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:01.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:01 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4240879384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:02 smithi152 ceph-mon[88640]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 793 op/s; 9535/360234 objects degraded (2.647%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:02.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:02 smithi102 ceph-mon[98773]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 793 op/s; 9535/360234 objects degraded (2.647%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:02.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:02 smithi102 ceph-mon[94535]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 793 op/s; 9535/360234 objects degraded (2.647%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 9535/360234 objects degraded (2.647%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:03.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:03.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 9535/360234 objects degraded (2.647%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:03.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 9535/360234 objects degraded (2.647%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:03.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:04.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:04 smithi152 ceph-mon[88640]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 724 op/s; 9535/357714 objects degraded (2.666%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:04.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/107340044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:04.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:04 smithi102 ceph-mon[94535]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 724 op/s; 9535/357714 objects degraded (2.666%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:04.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/107340044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:04.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:04 smithi102 ceph-mon[98773]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 724 op/s; 9535/357714 objects degraded (2.666%); 3.8 MiB/s, 60 objects/s recovering 2024-04-10T03:20:04.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/107340044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:06.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:06 smithi152 ceph-mon[88640]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 854 op/s; 9253/350541 objects degraded (2.640%); 5.2 MiB/s, 83 objects/s recovering 2024-04-10T03:20:06.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:06 smithi102 ceph-mon[98773]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 854 op/s; 9253/350541 objects degraded (2.640%); 5.2 MiB/s, 83 objects/s recovering 2024-04-10T03:20:06.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:06 smithi102 ceph-mon[94535]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 854 op/s; 9253/350541 objects degraded (2.640%); 5.2 MiB/s, 83 objects/s recovering 2024-04-10T03:20:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:07 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/41454380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:07.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:07.790 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:07 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/41454380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:07.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:07.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:07 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/41454380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:07.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:08.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:08 smithi152 ceph-mon[88640]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s; 9253/347298 objects degraded (2.664%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:20:08.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:08 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 9253/347298 objects degraded (2.664%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:08 smithi102 ceph-mon[94535]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s; 9253/347298 objects degraded (2.664%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:20:08.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:08 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 9253/347298 objects degraded (2.664%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:08 smithi102 ceph-mon[98773]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 733 op/s; 9253/347298 objects degraded (2.664%); 4.0 MiB/s, 63 objects/s recovering 2024-04-10T03:20:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:08 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 9253/347298 objects degraded (2.664%), 6 pgs degraded (PG_DEGRADED) 2024-04-10T03:20:09.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3463507743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:09.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3463507743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:09.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3463507743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:10.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:10 smithi152 ceph-mon[88640]: pgmap v373: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 857 op/s; 8951/339588 objects degraded (2.636%); 5.5 MiB/s, 88 objects/s recovering 2024-04-10T03:20:10.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:10 smithi152 ceph-mon[88640]: osdmap e119: 8 total, 8 up, 8 in 2024-04-10T03:20:10.693 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:10 smithi102 ceph-mon[98773]: pgmap v373: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 857 op/s; 8951/339588 objects degraded (2.636%); 5.5 MiB/s, 88 objects/s recovering 2024-04-10T03:20:10.693 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:10 smithi102 ceph-mon[98773]: osdmap e119: 8 total, 8 up, 8 in 2024-04-10T03:20:10.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:10 smithi102 ceph-mon[94535]: pgmap v373: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 857 op/s; 8951/339588 objects degraded (2.636%); 5.5 MiB/s, 88 objects/s recovering 2024-04-10T03:20:10.693 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:10 smithi102 ceph-mon[94535]: osdmap e119: 8 total, 8 up, 8 in 2024-04-10T03:20:11.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:10] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2024-04-10T03:20:11.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:11 smithi102 ceph-mon[98773]: osdmap e120: 8 total, 8 up, 8 in 2024-04-10T03:20:11.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:11 smithi102 ceph-mon[94535]: osdmap e120: 8 total, 8 up, 8 in 2024-04-10T03:20:11.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:11 smithi152 ceph-mon[88640]: osdmap e120: 8 total, 8 up, 8 in 2024-04-10T03:20:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:12 smithi152 ceph-mon[88640]: pgmap v376: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 8809/335442 objects degraded (2.626%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:20:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:12 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3790791270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:12.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[98773]: pgmap v376: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 8809/335442 objects degraded (2.626%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3790791270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[94535]: pgmap v376: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 8809/335442 objects degraded (2.626%); 5.7 MiB/s, 90 objects/s recovering 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3790791270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:12.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:13 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8809/335442 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:13.764 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:13 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8809/335442 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:13.764 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:13 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8809/335442 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:14.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:14 smithi102 ceph-mon[94535]: pgmap v377: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8809/333660 objects degraded (2.640%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:14.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3590557746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:14.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:14 smithi102 ceph-mon[98773]: pgmap v377: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8809/333660 objects degraded (2.640%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:14.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3590557746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:14.760 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:14 smithi152 ceph-mon[88640]: pgmap v377: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8809/333660 objects degraded (2.640%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:14.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3590557746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:16.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:16 smithi152 ceph-mon[88640]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8809/328248 objects degraded (2.684%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:16.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.538 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.538 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[94535]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8809/328248 objects degraded (2.684%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:16.538 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.538 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.539 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[98773]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8809/328248 objects degraded (2.684%); 3.5 MiB/s, 55 objects/s recovering 2024-04-10T03:20:16.539 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:16.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:16 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:20:16.540+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 8684/325311 objects degraded (2.669%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:17.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2743582653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 8684/325311 objects degraded (2.669%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2743582653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 8684/325311 objects degraded (2.669%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:20:17.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:17.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:17.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2743582653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8684/325311 objects degraded (2.669%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8684/325311 objects degraded (2.669%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8684/325311 objects degraded (2.669%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:19.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:19 smithi152 ceph-mon[88640]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 8684/323283 objects degraded (2.686%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:20:19.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:19 smithi102 ceph-mon[98773]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 8684/323283 objects degraded (2.686%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:20:19.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:19 smithi102 ceph-mon[94535]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 8684/323283 objects degraded (2.686%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:20:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3757473538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:20.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3757473538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:20.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3757473538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:21.259 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:20] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-10T03:20:21.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:21 smithi152 ceph-mon[88640]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8684/316665 objects degraded (2.742%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:20:21.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:21 smithi102 ceph-mon[98773]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8684/316665 objects degraded (2.742%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:20:21.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:21 smithi102 ceph-mon[94535]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8684/316665 objects degraded (2.742%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:20:22.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1870758691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:22.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1870758691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:22.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1870758691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:23.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:23 smithi102 ceph-mon[98773]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8523/313758 objects degraded (2.716%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:23.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:23 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8684/316665 objects degraded (2.742%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:23.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:23 smithi102 ceph-mon[94535]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8523/313758 objects degraded (2.716%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:23.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:23 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8684/316665 objects degraded (2.742%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:23 smithi152 ceph-mon[88640]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 8523/313758 objects degraded (2.716%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:23 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8684/316665 objects degraded (2.742%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:25.714 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:25 smithi102 ceph-mon[94535]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 8523/306882 objects degraded (2.777%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:25.714 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2334986432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:25.715 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:25 smithi102 ceph-mon[98773]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 8523/306882 objects degraded (2.777%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:25.715 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:25 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2334986432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:25 smithi152 ceph-mon[88640]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 8523/306882 objects degraded (2.777%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:25 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2334986432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:27.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:27 smithi152 ceph-mon[88640]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 8357/301557 objects degraded (2.771%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:27.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/908999524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:27.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:27 smithi102 ceph-mon[94535]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 8357/301557 objects degraded (2.771%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:27.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/908999524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:27.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:27 smithi102 ceph-mon[98773]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 8357/301557 objects degraded (2.771%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:27.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/908999524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:28 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8357/301557 objects degraded (2.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:28.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:28 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8357/301557 objects degraded (2.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:28.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:28 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8357/301557 objects degraded (2.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:29.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:29 smithi152 ceph-mon[88640]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8357/299343 objects degraded (2.792%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:29 smithi102 ceph-mon[94535]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8357/299343 objects degraded (2.792%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:29.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:29 smithi102 ceph-mon[98773]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8357/299343 objects degraded (2.792%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:30.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1433716081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:30.790 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1433716081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:30.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1433716081' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:30] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-10T03:20:31.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:31 smithi152 ceph-mon[88640]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8357/292725 objects degraded (2.855%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:31.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:31 smithi102 ceph-mon[94535]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8357/292725 objects degraded (2.855%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:31.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:20:31.558+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:31.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:31 smithi102 ceph-mon[98773]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 8357/292725 objects degraded (2.855%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:20:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:32 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:32 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:32 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:32 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:33 smithi152 ceph-mon[88640]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8216/289620 objects degraded (2.837%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:20:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2528160461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:33 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8216/289620 objects degraded (2.837%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:33.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[94535]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8216/289620 objects degraded (2.837%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2528160461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8216/289620 objects degraded (2.837%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[98773]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8216/289620 objects degraded (2.837%); 2.4 MiB/s, 38 objects/s recovering 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2528160461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8216/289620 objects degraded (2.837%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:33.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:35.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:35 smithi152 ceph-mon[88640]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8216/282909 objects degraded (2.904%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:35.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1070815943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:35.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:35 smithi102 ceph-mon[94535]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8216/282909 objects degraded (2.904%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:35.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1070815943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:35 smithi102 ceph-mon[98773]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8216/282909 objects degraded (2.904%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1070815943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:37.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:37 smithi152 ceph-mon[88640]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 8050/277779 objects degraded (2.898%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:20:37.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:37 smithi102 ceph-mon[94535]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 8050/277779 objects degraded (2.898%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:20:37.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:37 smithi102 ceph-mon[98773]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 8050/277779 objects degraded (2.898%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:20:38.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/446057844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:38.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:38 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 8050/277779 objects degraded (2.898%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/446057844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:38.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:38 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 8050/277779 objects degraded (2.898%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/446057844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:38.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:38 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 8050/277779 objects degraded (2.898%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:39.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:39 smithi152 ceph-mon[88640]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8050/275589 objects degraded (2.921%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:39.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:39 smithi102 ceph-mon[94535]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8050/275589 objects degraded (2.921%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:39.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:39 smithi102 ceph-mon[98773]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8050/275589 objects degraded (2.921%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:40.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:40 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2257742305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:40.794 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:40 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2257742305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:40.794 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:40 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2257742305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:40] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-10T03:20:41.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:41 smithi152 ceph-mon[88640]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 8050/268938 objects degraded (2.993%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:41.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:41 smithi102 ceph-mon[94535]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 8050/268938 objects degraded (2.993%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:41.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:41 smithi102 ceph-mon[98773]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 8050/268938 objects degraded (2.993%); 1.6 MiB/s, 25 objects/s recovering 2024-04-10T03:20:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:43 smithi152 ceph-mon[88640]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7869/266037 objects degraded (2.958%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:43 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7869/266037 objects degraded (2.958%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:43.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:43 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1936138601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[98773]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7869/266037 objects degraded (2.958%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7869/266037 objects degraded (2.958%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1936138601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[94535]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7869/266037 objects degraded (2.958%); 2.5 MiB/s, 40 objects/s recovering 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7869/266037 objects degraded (2.958%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:43.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:43 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1936138601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:45.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:45 smithi152 ceph-mon[88640]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7869/259638 objects degraded (3.031%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:20:45.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:45 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3889690678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:45.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:45 smithi102 ceph-mon[98773]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7869/259638 objects degraded (3.031%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:20:45.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:45 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3889690678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:45.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:45 smithi102 ceph-mon[94535]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7869/259638 objects degraded (3.031%); 1.8 MiB/s, 28 objects/s recovering 2024-04-10T03:20:45.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:45 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3889690678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:47.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:46 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:20:46.576+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:47 smithi152 ceph-mon[88640]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 7682/254601 objects degraded (3.017%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:20:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:47 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:47 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:47 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[94535]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 7682/254601 objects degraded (3.017%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:20:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:47.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[98773]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 7682/254601 objects degraded (3.017%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:20:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:20:47.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:47 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:20:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7682/254601 objects degraded (3.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:48 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/867972489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7682/254601 objects degraded (3.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/867972489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7682/254601 objects degraded (3.017%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:20:48.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:48 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/867972489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:49 smithi152 ceph-mon[88640]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7682/252384 objects degraded (3.044%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:49.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:49 smithi102 ceph-mon[94535]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7682/252384 objects degraded (3.044%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:49.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:49 smithi102 ceph-mon[98773]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 7682/252384 objects degraded (3.044%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:20:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:20:50] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-10T03:20:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:51 smithi152 ceph-mon[88640]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7682/245832 objects degraded (3.125%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:51.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/255631470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:51.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:51 smithi102 ceph-mon[98773]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7682/245832 objects degraded (3.125%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:51.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/255631470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:51.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:51 smithi102 ceph-mon[94535]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 7682/245832 objects degraded (3.125%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:20:51.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/255631470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:53 smithi152 ceph-mon[88640]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7474/242880 objects degraded (3.077%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:20:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:53 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7474/242880 objects degraded (3.077%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:53.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3083860331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[94535]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7474/242880 objects degraded (3.077%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:20:53.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7474/242880 objects degraded (3.077%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:53.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3083860331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:53.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[98773]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7474/242880 objects degraded (3.077%); 3.0 MiB/s, 47 objects/s recovering 2024-04-10T03:20:53.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7474/242880 objects degraded (3.077%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:53.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3083860331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:55.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:55 smithi102 ceph-mon[94535]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7474/236154 objects degraded (3.165%); 2.1 MiB/s, 32 objects/s recovering 2024-04-10T03:20:55.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:55 smithi102 ceph-mon[98773]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7474/236154 objects degraded (3.165%); 2.1 MiB/s, 32 objects/s recovering 2024-04-10T03:20:56.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:55 smithi152 ceph-mon[88640]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 7474/236154 objects degraded (3.165%); 2.1 MiB/s, 32 objects/s recovering 2024-04-10T03:20:56.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2029885953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:56.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2029885953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:57.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2029885953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:57.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:57 smithi102 ceph-mon[94535]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 7255/231027 objects degraded (3.140%); 3.2 MiB/s, 51 objects/s recovering 2024-04-10T03:20:57.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:57 smithi102 ceph-mon[94535]: osdmap e121: 8 total, 8 up, 8 in 2024-04-10T03:20:57.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:57 smithi102 ceph-mon[98773]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 7255/231027 objects degraded (3.140%); 3.2 MiB/s, 51 objects/s recovering 2024-04-10T03:20:57.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:57 smithi102 ceph-mon[98773]: osdmap e121: 8 total, 8 up, 8 in 2024-04-10T03:20:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:57 smithi152 ceph-mon[88640]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 7255/231027 objects degraded (3.140%); 3.2 MiB/s, 51 objects/s recovering 2024-04-10T03:20:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:57 smithi152 ceph-mon[88640]: osdmap e121: 8 total, 8 up, 8 in 2024-04-10T03:20:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:58 smithi152 ceph-mon[88640]: osdmap e122: 8 total, 8 up, 8 in 2024-04-10T03:20:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:58 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7255/231027 objects degraded (3.140%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:58 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3251159620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:59.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[94535]: osdmap e122: 8 total, 8 up, 8 in 2024-04-10T03:20:59.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7255/231027 objects degraded (3.140%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:59.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3251159620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:20:59.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[98773]: osdmap e122: 8 total, 8 up, 8 in 2024-04-10T03:20:59.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7255/231027 objects degraded (3.140%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-10T03:20:59.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:58 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3251159620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:20:59 smithi152 ceph-mon[88640]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7255/229014 objects degraded (3.168%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:21:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:20:59 smithi102 ceph-mon[94535]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7255/229014 objects degraded (3.168%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:21:00.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:20:59 smithi102 ceph-mon[98773]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 7255/229014 objects degraded (3.168%); 3.3 MiB/s, 53 objects/s recovering 2024-04-10T03:21:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:00] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-10T03:21:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:01 smithi152 ceph-mon[88640]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7043/222720 objects degraded (3.162%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:02.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:01 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1707173455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:02.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:01 smithi102 ceph-mon[98773]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7043/222720 objects degraded (3.162%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:02.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:01 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1707173455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:01 smithi102 ceph-mon[94535]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7043/222720 objects degraded (3.162%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:02.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:01 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1707173455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:02.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:01 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:21:01.593+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:02 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:02 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:02 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:03 smithi152 ceph-mon[88640]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7043/220764 objects degraded (3.190%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 7043/220764 objects degraded (3.190%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:03 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1396306600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:04.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[98773]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7043/220764 objects degraded (3.190%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 7043/220764 objects degraded (3.190%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1396306600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[94535]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7043/220764 objects degraded (3.190%); 3.4 MiB/s, 53 objects/s recovering 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 7043/220764 objects degraded (3.190%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:04.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:03 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1396306600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:06.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:05 smithi152 ceph-mon[88640]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7043/214233 objects degraded (3.288%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:21:06.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:05 smithi102 ceph-mon[94535]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7043/214233 objects degraded (3.288%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:21:06.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:05 smithi102 ceph-mon[98773]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7043/214233 objects degraded (3.288%); 1.7 MiB/s, 26 objects/s recovering 2024-04-10T03:21:07.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:06 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2102894943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:07.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:06 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2102894943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:07.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:06 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2102894943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:08.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:07 smithi152 ceph-mon[88640]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6886/209133 objects degraded (3.293%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:21:08.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:07 smithi102 ceph-mon[98773]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6886/209133 objects degraded (3.293%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:21:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:07 smithi102 ceph-mon[94535]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6886/209133 objects degraded (3.293%); 2.4 MiB/s, 37 objects/s recovering 2024-04-10T03:21:09.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:08 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6886/209133 objects degraded (3.293%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:09.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:08 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6886/209133 objects degraded (3.293%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:09.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:08 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6886/209133 objects degraded (3.293%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:09 smithi152 ceph-mon[88640]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6886/206991 objects degraded (3.327%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:21:10.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3358787508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:09 smithi102 ceph-mon[94535]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6886/206991 objects degraded (3.327%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:21:10.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3358787508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:09 smithi102 ceph-mon[98773]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 6886/206991 objects degraded (3.327%); 2.1 MiB/s, 34 objects/s recovering 2024-04-10T03:21:10.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3358787508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:10] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-10T03:21:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:11 smithi152 ceph-mon[88640]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6717/199593 objects degraded (3.365%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:12.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:11 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4055385698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:11 smithi102 ceph-mon[94535]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6717/199593 objects degraded (3.365%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:11 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4055385698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:12.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:11 smithi102 ceph-mon[98773]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 6717/199593 objects degraded (3.365%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:11 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4055385698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:13.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:12 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6717/199593 objects degraded (3.365%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:13.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:12 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6717/199593 objects degraded (3.365%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:13.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:12 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6717/199593 objects degraded (3.365%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:14.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:13 smithi152 ceph-mon[88640]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6717/197394 objects degraded (3.403%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:14.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:13 smithi102 ceph-mon[98773]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6717/197394 objects degraded (3.403%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:13 smithi102 ceph-mon[94535]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6717/197394 objects degraded (3.403%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:15.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3827684379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:15.040 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3827684379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:15.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3827684379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:16.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:15 smithi152 ceph-mon[88640]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6717/190755 objects degraded (3.521%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:16.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:15 smithi102 ceph-mon[94535]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6717/190755 objects degraded (3.521%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:16.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:15 smithi102 ceph-mon[98773]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6717/190755 objects degraded (3.521%); 1.7 MiB/s, 27 objects/s recovering 2024-04-10T03:21:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:16 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2788606529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:17.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:16 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2788606529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:16 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2788606529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:17.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:18.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:17 smithi152 ceph-mon[88640]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 6527/185754 objects degraded (3.514%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:21:18.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:17 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6527/185754 objects degraded (3.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:17 smithi102 ceph-mon[94535]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 6527/185754 objects degraded (3.514%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:21:18.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:17 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6527/185754 objects degraded (3.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:18.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:17 smithi102 ceph-mon[98773]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 6527/185754 objects degraded (3.514%); 2.7 MiB/s, 42 objects/s recovering 2024-04-10T03:21:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:17 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6527/185754 objects degraded (3.514%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:19.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:19.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:19 smithi152 ceph-mon[88640]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6527/183525 objects degraded (3.556%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:21:20.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4021129346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:20.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:19 smithi102 ceph-mon[94535]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6527/183525 objects degraded (3.556%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:21:20.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4021129346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:20.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:19 smithi102 ceph-mon[98773]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6527/183525 objects degraded (3.556%); 1.9 MiB/s, 29 objects/s recovering 2024-04-10T03:21:20.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4021129346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:21.275 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:20] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-10T03:21:22.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:21 smithi152 ceph-mon[88640]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6346/176493 objects degraded (3.596%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:22.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:21 smithi102 ceph-mon[98773]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6346/176493 objects degraded (3.596%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:22.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:21 smithi102 ceph-mon[94535]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 6346/176493 objects degraded (3.596%); 2.8 MiB/s, 44 objects/s recovering 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2299191083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:22 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6346/176493 objects degraded (3.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2299191083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6346/176493 objects degraded (3.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2299191083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:23.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:22 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6346/176493 objects degraded (3.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:23.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:21:23.126+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6346/174390 objects degraded (3.639%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:23 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6346/174390 objects degraded (3.639%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 6346/174390 objects degraded (3.639%); 1.9 MiB/s, 30 objects/s recovering 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:21:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:23 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:24 smithi152 ceph-mon[88640]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6346/172392 objects degraded (3.681%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:21:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:24 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:24 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:25.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/997355487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[94535]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6346/172392 objects degraded (3.681%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/997355487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[98773]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6346/172392 objects degraded (3.681%); 2.2 MiB/s, 34 objects/s recovering 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:25.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/997355487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:27.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:26 smithi152 ceph-mon[88640]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6138/165519 objects degraded (3.708%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:21:27.065 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:26 smithi102 ceph-mon[98773]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6138/165519 objects degraded (3.708%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:21:27.065 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:26 smithi102 ceph-mon[94535]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6138/165519 objects degraded (3.708%); 3.4 MiB/s, 54 objects/s recovering 2024-04-10T03:21:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3516770730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:27 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 6138/165519 objects degraded (3.708%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3516770730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:28.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:27 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 6138/165519 objects degraded (3.708%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:28.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3516770730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:28.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:27 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 6138/165519 objects degraded (3.708%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:29.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:28 smithi102 ceph-mon[98773]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6138/163437 objects degraded (3.756%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:28 smithi102 ceph-mon[94535]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6138/163437 objects degraded (3.756%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:29.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:28 smithi152 ceph-mon[88640]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 6138/163437 objects degraded (3.756%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:29 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2596677246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:29 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2596677246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:30.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:29 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2596677246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:31.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:30 smithi102 ceph-mon[98773]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 6138/161106 objects degraded (3.810%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:31.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:30] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-10T03:21:31.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:30 smithi102 ceph-mon[94535]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 6138/161106 objects degraded (3.810%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:30 smithi152 ceph-mon[88640]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 6138/161106 objects degraded (3.810%); 2.3 MiB/s, 36 objects/s recovering 2024-04-10T03:21:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[94535]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5923/151332 objects degraded (3.914%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1660500927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:33.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5923/151332 objects degraded (3.914%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[98773]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5923/151332 objects degraded (3.914%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1660500927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:33.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:32 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5923/151332 objects degraded (3.914%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:32 smithi152 ceph-mon[88640]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5923/151332 objects degraded (3.914%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:32 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1660500927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:32 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5923/151332 objects degraded (3.914%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:34.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:34.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:35.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:34 smithi152 ceph-mon[88640]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5923/149124 objects degraded (3.972%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:35.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:34 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2788099590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:35.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:34 smithi102 ceph-mon[94535]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5923/149124 objects degraded (3.972%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:35.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2788099590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:35.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:34 smithi102 ceph-mon[98773]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5923/149124 objects degraded (3.972%); 2.5 MiB/s, 39 objects/s recovering 2024-04-10T03:21:35.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:34 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2788099590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:37.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:36 smithi152 ceph-mon[88640]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 5674/141969 objects degraded (3.997%); 3.4 MiB/s, 55 objects/s recovering 2024-04-10T03:21:37.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:36 smithi102 ceph-mon[94535]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 5674/141969 objects degraded (3.997%); 3.4 MiB/s, 55 objects/s recovering 2024-04-10T03:21:37.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:36 smithi102 ceph-mon[98773]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 5674/141969 objects degraded (3.997%); 3.4 MiB/s, 55 objects/s recovering 2024-04-10T03:21:38.144 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:37 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2733584011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:38.144 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:37 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5674/141969 objects degraded (3.997%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:38.144 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:37 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2733584011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:38.144 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:37 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5674/141969 objects degraded (3.997%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:38.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:37 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2733584011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:38.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:37 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5674/141969 objects degraded (3.997%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:38.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:21:38.143+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:38 smithi152 ceph-mon[88640]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5674/139662 objects degraded (4.063%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:38 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:38 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:38 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[98773]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5674/139662 objects degraded (4.063%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[94535]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 5674/139662 objects degraded (4.063%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:38 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-10T03:21:40.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:39 smithi102 ceph-mon[94535]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 5674/137403 objects degraded (4.129%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:40.201 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:39 smithi102 ceph-mon[98773]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 5674/137403 objects degraded (4.129%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:40.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:39 smithi152 ceph-mon[88640]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 5674/137403 objects degraded (4.129%); 2.3 MiB/s, 38 objects/s recovering 2024-04-10T03:21:41.218 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:40 smithi102 ceph-mon[98773]: osdmap e123: 8 total, 8 up, 8 in 2024-04-10T03:21:41.218 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:40 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/778193140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:41.219 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:40 smithi102 ceph-mon[94535]: osdmap e123: 8 total, 8 up, 8 in 2024-04-10T03:21:41.219 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:40 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/778193140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:41.219 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:40] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-10T03:21:41.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:40 smithi152 ceph-mon[88640]: osdmap e123: 8 total, 8 up, 8 in 2024-04-10T03:21:41.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:40 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/778193140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:41.929 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:41 smithi102 ceph-mon[94535]: osdmap e124: 8 total, 8 up, 8 in 2024-04-10T03:21:41.930 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:41 smithi102 ceph-mon[94535]: pgmap v426: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 5224/128193 objects degraded (4.075%); 1.8 MiB/s, 87 objects/s recovering 2024-04-10T03:21:41.930 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:41 smithi102 ceph-mon[98773]: osdmap e124: 8 total, 8 up, 8 in 2024-04-10T03:21:41.930 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:41 smithi102 ceph-mon[98773]: pgmap v426: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 5224/128193 objects degraded (4.075%); 1.8 MiB/s, 87 objects/s recovering 2024-04-10T03:21:42.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:41 smithi152 ceph-mon[88640]: osdmap e124: 8 total, 8 up, 8 in 2024-04-10T03:21:42.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:41 smithi152 ceph-mon[88640]: pgmap v426: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 5224/128193 objects degraded (4.075%); 1.8 MiB/s, 87 objects/s recovering 2024-04-10T03:21:43.250 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:42 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2449968021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:43.250 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:42 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5224/128193 objects degraded (4.075%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:43.250 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:42 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2449968021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:43.250 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:42 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5224/128193 objects degraded (4.075%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:43.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:42 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2449968021' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:43.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:42 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5224/128193 objects degraded (4.075%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:43 smithi152 ceph-mon[88640]: pgmap v427: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5224/126225 objects degraded (4.139%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:44.290 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:43 smithi102 ceph-mon[98773]: pgmap v427: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5224/126225 objects degraded (4.139%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:44.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:43 smithi102 ceph-mon[94535]: pgmap v427: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5224/126225 objects degraded (4.139%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:46.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:46 smithi152 ceph-mon[88640]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 5224/120546 objects degraded (4.334%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:46.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3634598235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:46.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:46 smithi102 ceph-mon[98773]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 5224/120546 objects degraded (4.334%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:46.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3634598235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:46.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:46 smithi102 ceph-mon[94535]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 5224/120546 objects degraded (4.334%); 0 B/s, 56 objects/s recovering 2024-04-10T03:21:46.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3634598235' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:48.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:48 smithi152 ceph-mon[88640]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 5013/118350 objects degraded (4.236%); 0 B/s, 82 objects/s recovering 2024-04-10T03:21:48.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 5013/118350 objects degraded (4.236%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:48.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:48 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2641525297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:48.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[98773]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 5013/118350 objects degraded (4.236%); 0 B/s, 82 objects/s recovering 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 5013/118350 objects degraded (4.236%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2641525297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[94535]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 5013/118350 objects degraded (4.236%); 0 B/s, 82 objects/s recovering 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 5013/118350 objects degraded (4.236%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2641525297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:48.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:21:50.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:50 smithi102 ceph-mon[98773]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5013/116232 objects degraded (4.313%); 0 B/s, 72 objects/s recovering 2024-04-10T03:21:50.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:50 smithi102 ceph-mon[94535]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5013/116232 objects degraded (4.313%); 0 B/s, 72 objects/s recovering 2024-04-10T03:21:50.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:50 smithi152 ceph-mon[88640]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 5013/116232 objects degraded (4.313%); 0 B/s, 72 objects/s recovering 2024-04-10T03:21:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/445212942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:21:50] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2024-04-10T03:21:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/445212942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:51.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/445212942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:52.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:52 smithi102 ceph-mon[94535]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4811/106143 objects degraded (4.533%); 0 B/s, 40 objects/s recovering 2024-04-10T03:21:52.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:52 smithi102 ceph-mon[98773]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4811/106143 objects degraded (4.533%); 0 B/s, 40 objects/s recovering 2024-04-10T03:21:52.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:52 smithi152 ceph-mon[88640]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4811/106143 objects degraded (4.533%); 0 B/s, 40 objects/s recovering 2024-04-10T03:21:53.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:53 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4811/106143 objects degraded (4.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:53.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/859981092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:53.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:21:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:21:53.162+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:21:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:53 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4811/106143 objects degraded (4.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:53.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/859981092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:53.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:53 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4811/106143 objects degraded (4.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:53.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/859981092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[98773]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 4811/103881 objects degraded (4.631%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[94535]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 4811/103881 objects degraded (4.631%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:54 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:21:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:54 smithi152 ceph-mon[88640]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 4811/103881 objects degraded (4.631%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:54 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:21:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:54 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:21:56.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:56 smithi102 ceph-mon[98773]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 4811/96270 objects degraded (4.997%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:56.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2482551224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:56.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:56 smithi102 ceph-mon[94535]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 4811/96270 objects degraded (4.997%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:56.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2482551224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:56.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:56 smithi152 ceph-mon[88640]: pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 4811/96270 objects degraded (4.997%); 0 B/s, 34 objects/s recovering 2024-04-10T03:21:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2482551224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:58 smithi152 ceph-mon[88640]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4621/93993 objects degraded (4.916%); 0 B/s, 50 objects/s recovering 2024-04-10T03:21:58.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:58 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4621/93993 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:58.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:58 smithi102 ceph-mon[98773]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4621/93993 objects degraded (4.916%); 0 B/s, 50 objects/s recovering 2024-04-10T03:21:58.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:58 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4621/93993 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:58 smithi102 ceph-mon[94535]: pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4621/93993 objects degraded (4.916%); 0 B/s, 50 objects/s recovering 2024-04-10T03:21:58.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:58 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4621/93993 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:21:59.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:21:59 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2691043281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:59.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:21:59 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2691043281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:21:59.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:21:59 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2691043281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:00.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:00 smithi152 ceph-mon[88640]: pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4621/91713 objects degraded (5.039%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:00.540 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:00 smithi102 ceph-mon[94535]: pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4621/91713 objects degraded (5.039%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:00.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:00 smithi102 ceph-mon[98773]: pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4621/91713 objects degraded (5.039%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:01.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:01 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2051284208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:00] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2024-04-10T03:22:01.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:01 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2051284208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:01.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:01 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2051284208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:02.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:02 smithi152 ceph-mon[88640]: pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4412/81750 objects degraded (5.397%); 0 B/s, 50 objects/s recovering 2024-04-10T03:22:02.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:02 smithi102 ceph-mon[94535]: pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4412/81750 objects degraded (5.397%); 0 B/s, 50 objects/s recovering 2024-04-10T03:22:02.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:02 smithi102 ceph-mon[98773]: pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4412/81750 objects degraded (5.397%); 0 B/s, 50 objects/s recovering 2024-04-10T03:22:03.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:03 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4412/81750 objects degraded (5.397%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:03.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:03.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:03 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4412/81750 objects degraded (5.397%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:03.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:03 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4412/81750 objects degraded (5.397%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:03.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:04.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:04 smithi152 ceph-mon[88640]: pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 4412/79440 objects degraded (5.554%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:04.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/396256984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:04.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:04 smithi102 ceph-mon[94535]: pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 4412/79440 objects degraded (5.554%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:04.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/396256984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:04.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:04 smithi102 ceph-mon[98773]: pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 4412/79440 objects degraded (5.554%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:04.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/396256984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:06.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:06 smithi152 ceph-mon[88640]: pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 4412/74592 objects degraded (5.915%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:06.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:06 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3296514646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:06 smithi102 ceph-mon[94535]: pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 4412/74592 objects degraded (5.915%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:06.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:06 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3296514646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:06.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:06 smithi102 ceph-mon[98773]: pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 4412/74592 objects degraded (5.915%); 0 B/s, 33 objects/s recovering 2024-04-10T03:22:06.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:06 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3296514646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:08.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:08 smithi152 ceph-mon[88640]: pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4195/69231 objects degraded (6.059%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:08.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:08 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4195/69231 objects degraded (6.059%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:08.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:08 smithi102 ceph-mon[98773]: pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4195/69231 objects degraded (6.059%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:08.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:08 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4195/69231 objects degraded (6.059%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:08.541 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:08 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:22:08.179+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:22:08.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:08 smithi102 ceph-mon[94535]: pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4195/69231 objects degraded (6.059%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:08.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:08 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4195/69231 objects degraded (6.059%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:09 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:09 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:09 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:22:09.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3543823378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3543823378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-10T03:22:09.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3543823378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:10.441 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:10 smithi102 ceph-mon[94535]: pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4195/66903 objects degraded (6.270%); 0 B/s, 35 objects/s recovering 2024-04-10T03:22:10.441 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:10 smithi102 ceph-mon[98773]: pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4195/66903 objects degraded (6.270%); 0 B/s, 35 objects/s recovering 2024-04-10T03:22:10.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:10 smithi152 ceph-mon[88640]: pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4195/66903 objects degraded (6.270%); 0 B/s, 35 objects/s recovering 2024-04-10T03:22:11.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:11 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1963191296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:10] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2024-04-10T03:22:11.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:11 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1963191296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:11.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:11 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1963191296' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:12.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:12 smithi152 ceph-mon[88640]: pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 3932/57042 objects degraded (6.893%); 0 B/s, 57 objects/s recovering 2024-04-10T03:22:12.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:12 smithi102 ceph-mon[98773]: pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 3932/57042 objects degraded (6.893%); 0 B/s, 57 objects/s recovering 2024-04-10T03:22:12.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:12 smithi102 ceph-mon[94535]: pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 3932/57042 objects degraded (6.893%); 0 B/s, 57 objects/s recovering 2024-04-10T03:22:13.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:13 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3932/57042 objects degraded (6.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:13.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:13 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3932/57042 objects degraded (6.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:13.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:13 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3932/57042 objects degraded (6.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:14.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:14 smithi152 ceph-mon[88640]: pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3932/54741 objects degraded (7.183%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:14.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2610216857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:14.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:14 smithi102 ceph-mon[98773]: pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3932/54741 objects degraded (7.183%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:14.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2610216857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:14 smithi102 ceph-mon[94535]: pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3932/54741 objects degraded (7.183%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:14.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2610216857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:16.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:16 smithi152 ceph-mon[88640]: pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3932/50193 objects degraded (7.834%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:16.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:16 smithi102 ceph-mon[98773]: pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3932/50193 objects degraded (7.834%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:16 smithi102 ceph-mon[94535]: pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3932/50193 objects degraded (7.834%); 0 B/s, 39 objects/s recovering 2024-04-10T03:22:17.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2858599822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:17.540 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2858599822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:17.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2858599822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:18 smithi152 ceph-mon[88640]: pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 931 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3630/44754 objects degraded (8.111%); 0 B/s, 65 objects/s recovering 2024-04-10T03:22:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:18 smithi152 ceph-mon[88640]: osdmap e125: 8 total, 8 up, 8 in 2024-04-10T03:22:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:18 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3630/44754 objects degraded (8.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:18.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[94535]: pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 931 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3630/44754 objects degraded (8.111%); 0 B/s, 65 objects/s recovering 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[94535]: osdmap e125: 8 total, 8 up, 8 in 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3630/44754 objects degraded (8.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[98773]: pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 931 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 3630/44754 objects degraded (8.111%); 0 B/s, 65 objects/s recovering 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[98773]: osdmap e125: 8 total, 8 up, 8 in 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3630/44754 objects degraded (8.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:18.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:19.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:19 smithi152 ceph-mon[88640]: osdmap e126: 8 total, 8 up, 8 in 2024-04-10T03:22:19.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/996890419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:19.540 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:19 smithi102 ceph-mon[94535]: osdmap e126: 8 total, 8 up, 8 in 2024-04-10T03:22:19.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/996890419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:19.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:19 smithi102 ceph-mon[98773]: osdmap e126: 8 total, 8 up, 8 in 2024-04-10T03:22:19.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/996890419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:20.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:20 smithi152 ceph-mon[88640]: pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 886 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3630/42597 objects degraded (8.522%); 0 B/s, 37 objects/s recovering 2024-04-10T03:22:20.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:20 smithi102 ceph-mon[94535]: pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 886 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3630/42597 objects degraded (8.522%); 0 B/s, 37 objects/s recovering 2024-04-10T03:22:20.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:20 smithi102 ceph-mon[98773]: pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 886 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 3630/42597 objects degraded (8.522%); 0 B/s, 37 objects/s recovering 2024-04-10T03:22:21.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:20] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2024-04-10T03:22:22.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:22 smithi152 ceph-mon[88640]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 3275/34452 objects degraded (9.506%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:22.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3000799412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:22.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:22 smithi102 ceph-mon[98773]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 3275/34452 objects degraded (9.506%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:22.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3000799412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:22.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:22 smithi102 ceph-mon[94535]: pgmap v448: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 717 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 3275/34452 objects degraded (9.506%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:22.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3000799412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:23.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:23 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3275/34452 objects degraded (9.506%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:23.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:23 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3275/34452 objects degraded (9.506%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:23.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:23 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3275/34452 objects degraded (9.506%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:24.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:24 smithi152 ceph-mon[88640]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 671 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3275/32262 objects degraded (10.151%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:24.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:24 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:24.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/49991875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:24.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[94535]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 671 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3275/32262 objects degraded (10.151%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:24.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:24.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/49991875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:24.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[98773]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 671 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3275/32262 objects degraded (10.151%); 0 B/s, 82 objects/s recovering 2024-04-10T03:22:24.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:24.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/49991875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:26.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:26 smithi152 ceph-mon[88640]: pgmap v450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 576 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3275/27699 objects degraded (11.824%); 0 B/s, 44 objects/s recovering 2024-04-10T03:22:26.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:26 smithi102 ceph-mon[98773]: pgmap v450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 576 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3275/27699 objects degraded (11.824%); 0 B/s, 44 objects/s recovering 2024-04-10T03:22:26.543 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:26 smithi102 ceph-mon[94535]: pgmap v450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 576 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3275/27699 objects degraded (11.824%); 0 B/s, 44 objects/s recovering 2024-04-10T03:22:27.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/408764110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:27.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/408764110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:27.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/408764110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:28.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:28 smithi152 ceph-mon[88640]: pgmap v451: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 473 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3078/22737 objects degraded (13.537%); 0 B/s, 56 objects/s recovering 2024-04-10T03:22:28.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:28 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3078/22737 objects degraded (13.537%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:28.540 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:28 smithi102 ceph-mon[98773]: pgmap v451: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 473 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3078/22737 objects degraded (13.537%); 0 B/s, 56 objects/s recovering 2024-04-10T03:22:28.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:28 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3078/22737 objects degraded (13.537%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:28.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:28 smithi102 ceph-mon[94535]: pgmap v451: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 473 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3078/22737 objects degraded (13.537%); 0 B/s, 56 objects/s recovering 2024-04-10T03:22:28.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:28 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3078/22737 objects degraded (13.537%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:29 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:29 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:29 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2409947449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2409947449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:30.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:29 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2409947449' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:30 smithi152 ceph-mon[88640]: pgmap v452: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 430 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3078/20682 objects degraded (14.883%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:31.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[98773]: pgmap v452: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 430 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3078/20682 objects degraded (14.883%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:30] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.43.0" 2024-04-10T03:22:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[94535]: pgmap v452: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 430 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3078/20682 objects degraded (14.883%); 0 B/s, 51 objects/s recovering 2024-04-10T03:22:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:31.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: pgmap v453: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 286 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3078/13764 objects degraded (22.363%); 0 B/s, 46 objects/s recovering 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:32.200 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1513384284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: pgmap v453: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 286 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3078/13764 objects degraded (22.363%); 0 B/s, 46 objects/s recovering 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:32.201 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:31 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1513384284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: pgmap v453: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 286 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3078/13764 objects degraded (22.363%); 0 B/s, 46 objects/s recovering 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:31 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1513384284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:32.881 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:32.884 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[94535]: Upgrade: osd.2 is safe to restart 2024-04-10T03:22:32.885 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 3078/13764 objects degraded (22.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:32.885 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:32.885 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[98773]: Upgrade: osd.2 is safe to restart 2024-04-10T03:22:32.885 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:32 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 3078/13764 objects degraded (22.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:32 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-10T03:22:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:32 smithi152 ceph-mon[88640]: Upgrade: osd.2 is safe to restart 2024-04-10T03:22:33.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:32 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 3078/13764 objects degraded (22.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[98773]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 245 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2886/11787 objects degraded (24.485%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[94535]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 245 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2886/11787 objects degraded (24.485%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:34.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:22:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:33 smithi152 ceph-mon[88640]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 245 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 2886/11787 objects degraded (24.485%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-10T03:22:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:34.643 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Removed 121478 objects 2024-04-10T03:22:34.643 INFO:tasks.radosbench.radosbench.0.smithi102.stdout:Clean up completed and total clean up time :157.04 2024-04-10T03:22:35.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[94535]: Upgrade: Updating osd.2 2024-04-10T03:22:35.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[94535]: Deploying daemon osd.2 on smithi102 2024-04-10T03:22:35.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3227850445' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:35.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[94535]: pgmap v455: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 111 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2886/5382 objects degraded (53.623%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:35.029 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[98773]: Upgrade: Updating osd.2 2024-04-10T03:22:35.029 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[98773]: Deploying daemon osd.2 on smithi102 2024-04-10T03:22:35.029 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3227850445' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:35.029 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:34 smithi102 ceph-mon[98773]: pgmap v455: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 111 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2886/5382 objects degraded (53.623%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:35.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:34 smithi152 ceph-mon[88640]: Upgrade: Updating osd.2 2024-04-10T03:22:35.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:34 smithi152 ceph-mon[88640]: Deploying daemon osd.2 on smithi102 2024-04-10T03:22:35.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:34 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3227850445' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:35.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:34 smithi152 ceph-mon[88640]: pgmap v455: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 111 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2886/5382 objects degraded (53.623%); 0 B/s, 32 objects/s recovering 2024-04-10T03:22:36.701 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-04-10T03:22:36.702 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-04-10T03:22:37.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:36 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2372167665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:37.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:36 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2372167665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:37.063 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:36 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2372167665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:37.603 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:22:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:37 smithi102 ceph-mon[94535]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 24 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2395/1212 objects degraded (197.607%); 0 B/s, 74 objects/s recovering 2024-04-10T03:22:38.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:37 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 2395/1212 objects degraded (197.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:38.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:37 smithi102 ceph-mon[98773]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 24 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2395/1212 objects degraded (197.607%); 0 B/s, 74 objects/s recovering 2024-04-10T03:22:38.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:37 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 2395/1212 objects degraded (197.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:38.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:37 smithi152 ceph-mon[88640]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 24 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 2395/1212 objects degraded (197.607%); 0 B/s, 74 objects/s recovering 2024-04-10T03:22:38.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:37 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 2395/1212 objects degraded (197.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:38 smithi152 ceph-mon[88640]: osdmap e127: 8 total, 8 up, 8 in 2024-04-10T03:22:39.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:38 smithi152 ceph-mon[88640]: pgmap v458: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2395/357 objects degraded (670.868%); 0 B/s, 69 objects/s recovering 2024-04-10T03:22:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:38 smithi102 ceph-mon[94535]: osdmap e127: 8 total, 8 up, 8 in 2024-04-10T03:22:39.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:38 smithi102 ceph-mon[94535]: pgmap v458: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2395/357 objects degraded (670.868%); 0 B/s, 69 objects/s recovering 2024-04-10T03:22:39.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:38 smithi102 ceph-mon[98773]: osdmap e127: 8 total, 8 up, 8 in 2024-04-10T03:22:39.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:38 smithi102 ceph-mon[98773]: pgmap v458: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 6.4 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2395/357 objects degraded (670.868%); 0 B/s, 69 objects/s recovering 2024-04-10T03:22:39.292 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:39 smithi102 systemd[1]: Stopping Ceph osd.2 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:22:40.191 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:39 smithi102 ceph-mon[94535]: osdmap e128: 8 total, 8 up, 8 in 2024-04-10T03:22:40.192 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:39 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2818377277' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:40.192 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:39 smithi102 ceph-mon[98773]: osdmap e128: 8 total, 8 up, 8 in 2024-04-10T03:22:40.192 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:39 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2818377277' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:40.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:39 smithi152 ceph-mon[88640]: osdmap e128: 8 total, 8 up, 8 in 2024-04-10T03:22:40.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:39 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2818377277' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:41.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:40 smithi152 ceph-mon[88640]: pgmap v460: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 2395/72 objects degraded (3326.389%); 0 B/s, 61 objects/s recovering 2024-04-10T03:22:41.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:40 smithi102 ceph-mon[94535]: pgmap v460: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 2395/72 objects degraded (3326.389%); 0 B/s, 61 objects/s recovering 2024-04-10T03:22:41.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:40 smithi102 ceph-mon[98773]: pgmap v460: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 2395/72 objects degraded (3326.389%); 0 B/s, 61 objects/s recovering 2024-04-10T03:22:41.294 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:40] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.43.0" 2024-04-10T03:22:42.173 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:41 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1080781532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:42.173 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:41 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1080781532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:42.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:41 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1080781532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:42.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[55292]: 2024-04-10T03:22:42.468+0000 7fb770a10700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:22:42.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[55292]: 2024-04-10T03:22:42.468+0000 7fb770a10700 -1 osd.2 128 *** Got signal Terminated *** 2024-04-10T03:22:42.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[55292]: 2024-04-10T03:22:42.468+0000 7fb770a10700 -1 osd.2 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:22:43.116 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[94535]: osd.2 marked itself down and dead 2024-04-10T03:22:43.116 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 2395/72 objects degraded (3326.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:43.116 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[94535]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 875/72 objects degraded (1215.278%); 0 B/s, 251 objects/s recovering 2024-04-10T03:22:43.117 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[98773]: osd.2 marked itself down and dead 2024-04-10T03:22:43.117 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 2395/72 objects degraded (3326.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:43.117 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:42 smithi102 ceph-mon[98773]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 875/72 objects degraded (1215.278%); 0 B/s, 251 objects/s recovering 2024-04-10T03:22:43.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:42 smithi152 ceph-mon[88640]: osd.2 marked itself down and dead 2024-04-10T03:22:43.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:42 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 2395/72 objects degraded (3326.389%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-10T03:22:43.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:42 smithi152 ceph-mon[88640]: pgmap v461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 875/72 objects degraded (1215.278%); 0 B/s, 251 objects/s recovering 2024-04-10T03:22:43.658 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:43 smithi102 podman[125173]: 2024-04-10 03:22:43.374519661 +0000 UTC m=+1.285815294 container died f10dfba3072f4dabcdebca1f77a93fdacc0fd29f5e0414dfb8b4d9b5c4aa0c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux ) 2024-04-10T03:22:43.896 INFO:teuthology.orchestra.run.smithi102.stderr:pool 'unique_pool_3' removed 2024-04-10T03:22:43.919 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[94535]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:22:43.919 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[94535]: osdmap e129: 8 total, 7 up, 8 in 2024-04-10T03:22:43.919 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/858121686' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:43.919 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:43.920 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[98773]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:22:43.920 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[98773]: osdmap e129: 8 total, 7 up, 8 in 2024-04-10T03:22:43.920 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/858121686' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:43.920 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:43 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:43.920 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:43 smithi102 podman[125173]: 2024-04-10 03:22:43.762337589 +0000 UTC m=+1.673633228 container cleanup f10dfba3072f4dabcdebca1f77a93fdacc0fd29f5e0414dfb8b4d9b5c4aa0c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, ceph=True) 2024-04-10T03:22:43.920 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:43 smithi102 bash[125173]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2 2024-04-10T03:22:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:43 smithi152 ceph-mon[88640]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:22:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:43 smithi152 ceph-mon[88640]: osdmap e129: 8 total, 7 up, 8 in 2024-04-10T03:22:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:43 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/858121686' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:44.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:43 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-10T03:22:44.291 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:44 smithi102 podman[125227]: 2024-04-10 03:22:44.031905579 +0000 UTC m=+0.652292888 container remove f10dfba3072f4dabcdebca1f77a93fdacc0fd29f5e0414dfb8b4d9b5c4aa0c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:22:44.616 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-10T03:22:44.616 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-04-10T03:22:44.616 DEBUG:teuthology.parallel:result is None 2024-04-10T03:22:45.177 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[94535]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:22:45.177 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[94535]: osdmap e130: 8 total, 7 up, 8 in 2024-04-10T03:22:45.177 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3060840423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:45.177 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[94535]: pgmap v464: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:45.178 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[98773]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:22:45.178 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[98773]: osdmap e130: 8 total, 7 up, 8 in 2024-04-10T03:22:45.178 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3060840423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:45.178 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:44 smithi102 ceph-mon[98773]: pgmap v464: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:45.178 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:44.93232983 +0000 UTC m=+0.026239780 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:22:45.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:44 smithi152 ceph-mon[88640]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-10T03:22:45.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:44 smithi152 ceph-mon[88640]: osdmap e130: 8 total, 7 up, 8 in 2024-04-10T03:22:45.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:44 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3060840423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:45.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:44 smithi152 ceph-mon[88640]: pgmap v464: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 6 active+undersized, 64 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:45.478 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:45.178554317 +0000 UTC m=+0.272464309 container create 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:22:45.478 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:45.39828016 +0000 UTC m=+0.492190119 container init 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403) 2024-04-10T03:22:45.478 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:45.401669137 +0000 UTC m=+0.495579096 container start 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-10T03:22:45.478 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:45.481223095 +0000 UTC m=+0.575133053 container attach 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-10T03:22:45.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:45 smithi102 podman[125358]: 2024-04-10 03:22:45.724617648 +0000 UTC m=+0.818527590 container died 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403) 2024-04-10T03:22:46.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/942761485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:46.790 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/942761485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:46.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/942761485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:47.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:47 smithi102 ceph-mon[94535]: pgmap v465: 73 pgs: 1 active+undersized+degraded, 15 active+undersized, 57 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:47.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:47 smithi102 ceph-mon[98773]: pgmap v465: 73 pgs: 1 active+undersized+degraded, 15 active+undersized, 57 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:47.541 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 podman[125384]: 2024-04-10 03:22:47.178030278 +0000 UTC m=+1.448509596 container remove 9fd05559978f5706c95fffa8ba60e751419a9e34493b511b5903fe2cc39eb260 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-10T03:22:47.541 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.2.service: Deactivated successfully. 2024-04-10T03:22:47.542 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 systemd[1]: Stopped Ceph osd.2 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:22:47.542 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.2.service: Consumed 27min 21.267s CPU time. 2024-04-10T03:22:47.542 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 systemd[1]: Starting Ceph osd.2 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:22:47.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:47 smithi152 ceph-mon[88640]: pgmap v465: 73 pgs: 1 active+undersized+degraded, 15 active+undersized, 57 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:22:47.929 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 podman[125515]: 2024-04-10 03:22:47.535998623 +0000 UTC m=+0.017794844 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:22:47.929 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 podman[125515]: 2024-04-10 03:22:47.662499127 +0000 UTC m=+0.144295356 container create b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-10T03:22:47.929 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 podman[125515]: 2024-04-10 03:22:47.932491075 +0000 UTC m=+0.414287295 container init b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, ceph=True) 2024-04-10T03:22:48.291 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:47 smithi102 podman[125515]: 2024-04-10 03:22:47.935847325 +0000 UTC m=+0.417643546 container start b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-10T03:22:48.291 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 podman[125515]: 2024-04-10 03:22:48.040856718 +0000 UTC m=+0.522652948 container attach b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-10T03:22:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:48 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:22:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:48.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:48.790 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:22:48.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:48.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:22:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:48.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:22:48.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-10T03:22:48.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-10T03:22:48.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 bash[125515]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate[125531]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-10T03:22:48.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:48 smithi102 podman[125515]: 2024-04-10 03:22:48.516271378 +0000 UTC m=+0.998067607 container died b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-10T03:22:49.717 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2642528570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:49.717 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:49 smithi102 ceph-mon[94535]: pgmap v466: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:49.718 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2642528570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:49.718 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:49 smithi102 ceph-mon[98773]: pgmap v466: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:49.718 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 podman[125671]: 2024-04-10 03:22:49.36541919 +0000 UTC m=+0.843522192 container remove b93f70479e22faf2edea563f71cb9392961b6b7d340c990d774ac06dfcbdf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-10T03:22:49.718 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 podman[125734]: 2024-04-10 03:22:49.490521578 +0000 UTC m=+0.018775129 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:22:49.718 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 podman[125734]: 2024-04-10 03:22:49.588141378 +0000 UTC m=+0.116394915 container create 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, ceph=True) 2024-04-10T03:22:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2642528570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:49.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:49 smithi152 ceph-mon[88640]: pgmap v466: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:49.971 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 podman[125734]: 2024-04-10 03:22:49.84180487 +0000 UTC m=+0.370058417 container init 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:22:49.971 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 podman[125734]: 2024-04-10 03:22:49.845132826 +0000 UTC m=+0.373386364 container start 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-10T03:22:49.971 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 ceph-osd[125752]: -- 172.21.15.102:0/1632507288 <== mon.0 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559bf04b7680 con 0x559bf0480c00 2024-04-10T03:22:50.291 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 bash[125734]: 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 2024-04-10T03:22:50.291 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:49 smithi102 systemd[1]: Started Ceph osd.2 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:22:50.791 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:22:50.436+0000 7feb3650e7c0 -1 Falling back to public interface 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:22:50] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[94535]: pgmap v467: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:51.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2250984926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[98773]: pgmap v467: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2250984926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:51 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:51 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:51 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:51 smithi152 ceph-mon[88640]: pgmap v467: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:51.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2250984926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:53.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:52 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:22:53.040 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:52 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:22:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:52 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:22:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:53 smithi102 ceph-mon[98773]: pgmap v468: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:53.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:53 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3532288534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:53 smithi102 ceph-mon[94535]: pgmap v468: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:53.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:53 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3532288534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:53 smithi152 ceph-mon[88640]: pgmap v468: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:54.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:53 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3532288534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:54 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:54 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:54 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.891 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[98773]: pgmap v469: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:55.891 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.891 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.891 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1336201239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:55.891 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[94535]: pgmap v469: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:55.892 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.892 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:55.892 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:55 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1336201239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:56.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:55 smithi152 ceph-mon[88640]: pgmap v469: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:56.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:56.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:55 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:56.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:55 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1336201239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:56.830 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:56 smithi102 ceph-mon[94535]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:22:56.831 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:56 smithi102 ceph-mon[98773]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:22:56.831 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:56 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:22:56.550+0000 7feb3650e7c0 -1 osd.2 128 log_to_monitors true 2024-04-10T03:22:56.831 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:22:56 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:22:56.833+0000 7feb27966700 -1 osd.2 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:22:57.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:56 smithi152 ceph-mon[88640]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-10T03:22:58.186 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[98773]: pgmap v470: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:58.186 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[98773]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:22:58.186 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[98773]: osdmap e131: 8 total, 7 up, 8 in 2024-04-10T03:22:58.187 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[98773]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:22:58.187 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[94535]: pgmap v470: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:58.187 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[94535]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:22:58.187 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[94535]: osdmap e131: 8 total, 7 up, 8 in 2024-04-10T03:22:58.187 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:57 smithi102 ceph-mon[94535]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:22:58.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:57 smithi152 ceph-mon[88640]: pgmap v470: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:22:58.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:57 smithi152 ceph-mon[88640]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-10T03:22:58.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:57 smithi152 ceph-mon[88640]: osdmap e131: 8 total, 7 up, 8 in 2024-04-10T03:22:58.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:57 smithi152 ceph-mon[88640]: from='osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:22:58.791 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:22:58.681+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862] boot 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: osdmap e132: 8 total, 8 up, 8 in 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3783267824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:22:59.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:58 smithi152 ceph-mon[88640]: osdmap e133: 8 total, 8 up, 8 in 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862] boot 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: osdmap e132: 8 total, 8 up, 8 in 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3783267824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:59.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[98773]: osdmap e133: 8 total, 8 up, 8 in 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: osd.2 [v2:172.21.15.102:6818/4061930862,v1:172.21.15.102:6819/4061930862] boot 2024-04-10T03:22:59.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: osdmap e132: 8 total, 8 up, 8 in 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3783267824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:22:59.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:58 smithi102 ceph-mon[94535]: osdmap e133: 8 total, 8 up, 8 in 2024-04-10T03:23:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:59 smithi152 ceph-mon[88640]: pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:59 smithi152 ceph-mon[88640]: pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:59 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:59 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-10T03:23:00.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:22:59 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[94535]: pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[94535]: pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[98773]: pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[98773]: pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 15 active+undersized, 55 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:00.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-10T03:23:00.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:22:59 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-10T03:23:01.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:00 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/357432036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:01.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:00 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/357432036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:01.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:00] "GET /metrics HTTP/1.1" 200 33988 "" "Prometheus/2.43.0" 2024-04-10T03:23:01.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:00 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/357432036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:02.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:01 smithi152 ceph-mon[88640]: pgmap v476: 73 pgs: 2 peering, 5 active+undersized, 66 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:02.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:01 smithi152 ceph-mon[88640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-10T03:23:02.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:01 smithi102 ceph-mon[94535]: pgmap v476: 73 pgs: 2 peering, 5 active+undersized, 66 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:02.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:01 smithi102 ceph-mon[94535]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-10T03:23:02.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:01 smithi102 ceph-mon[98773]: pgmap v476: 73 pgs: 2 peering, 5 active+undersized, 66 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:02.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:01 smithi102 ceph-mon[98773]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-10T03:23:03.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1624105004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:03.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1624105004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:03.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1624105004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:04.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:03 smithi152 ceph-mon[88640]: pgmap v477: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:04.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:04.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:04.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[94535]: pgmap v477: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:04.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:04.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:04.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[98773]: pgmap v477: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:04.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:04.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:05.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:04 smithi152 ceph-mon[88640]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:05.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:04 smithi102 ceph-mon[98773]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:04 smithi102 ceph-mon[94535]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:06.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:05 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2061238208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:06.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:05 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2061238208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:06.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:05 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2061238208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:07.162 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:06 smithi152 ceph-mon[88640]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:07.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:06 smithi102 ceph-mon[94535]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:07.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:06 smithi102 ceph-mon[98773]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:08.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:07 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2074546944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:08.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:07 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2074546944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:08.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:07 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2074546944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:09.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:08 smithi152 ceph-mon[88640]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:09.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:08 smithi102 ceph-mon[94535]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:09.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:08 smithi102 ceph-mon[98773]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:10.192 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/369278187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:10.192 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/369278187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:10.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/369278187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:11.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:10 smithi152 ceph-mon[88640]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:11.290 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:10 smithi102 ceph-mon[94535]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:11.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:10 smithi102 ceph-mon[98773]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:11.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:10] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:23:12.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:11 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3721685064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:12.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:11 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3721685064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:12.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:11 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3721685064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:13.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:12 smithi152 ceph-mon[88640]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:13.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:12 smithi102 ceph-mon[98773]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:13.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:12 smithi102 ceph-mon[94535]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:14 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:14 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:14 smithi152 ceph-mon[88640]: Upgrade: osd.3 is safe to restart 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[94535]: Upgrade: osd.3 is safe to restart 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-10T03:23:14.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:14 smithi102 ceph-mon[98773]: Upgrade: osd.3 is safe to restart 2024-04-10T03:23:15.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:15 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1192912610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:15.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:15 smithi152 ceph-mon[88640]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:15.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1192912610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:15.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:15 smithi102 ceph-mon[94535]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:15.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1192912610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:15.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:15 smithi102 ceph-mon[98773]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[94535]: Upgrade: Updating osd.3 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[94535]: Deploying daemon osd.3 on smithi102 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[98773]: Upgrade: Updating osd.3 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:16.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:23:16.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:16.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:16 smithi102 ceph-mon[98773]: Deploying daemon osd.3 on smithi102 2024-04-10T03:23:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:16 smithi152 ceph-mon[88640]: Upgrade: Updating osd.3 2024-04-10T03:23:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-10T03:23:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:16 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:16.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:16 smithi152 ceph-mon[88640]: Deploying daemon osd.3 on smithi102 2024-04-10T03:23:17.516 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:17 smithi102 ceph-mon[94535]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:17.517 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:17 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/837328087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:17.517 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:17 smithi102 ceph-mon[98773]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:17.517 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:17 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/837328087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:17.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:17 smithi152 ceph-mon[88640]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:17.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:17 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/837328087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:18.699 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:18.700 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:18.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:19.041 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:18 smithi102 systemd[1]: Stopping Ceph osd.3 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:23:19.340 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[61165]: 2024-04-10T03:23:19.054+0000 7eff47088700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:23:19.340 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[61165]: 2024-04-10T03:23:19.054+0000 7eff47088700 -1 osd.3 133 *** Got signal Terminated *** 2024-04-10T03:23:19.340 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[61165]: 2024-04-10T03:23:19.054+0000 7eff47088700 -1 osd.3 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:23:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:19 smithi152 ceph-mon[88640]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2834631198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:19.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:19 smithi152 ceph-mon[88640]: osd.3 marked itself down and dead 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[94535]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2834631198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[94535]: osd.3 marked itself down and dead 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[98773]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2834631198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:19.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:19 smithi102 ceph-mon[98773]: osd.3 marked itself down and dead 2024-04-10T03:23:20.287 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:20 smithi102 podman[130134]: 2024-04-10 03:23:20.003592795 +0000 UTC m=+1.079415464 container died 6f7249bbf2b8fd3640be6b3686002a44362e26a106d339db9c5120a69280af48 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:23:20.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:20 smithi102 ceph-mon[94535]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:23:20.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:20 smithi102 ceph-mon[94535]: osdmap e134: 8 total, 7 up, 8 in 2024-04-10T03:23:20.541 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:20 smithi102 podman[130134]: 2024-04-10 03:23:20.446974396 +0000 UTC m=+1.522797136 container cleanup 6f7249bbf2b8fd3640be6b3686002a44362e26a106d339db9c5120a69280af48 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060) 2024-04-10T03:23:20.541 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:20 smithi102 bash[130134]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3 2024-04-10T03:23:20.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:20 smithi102 ceph-mon[98773]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:23:20.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:20 smithi102 ceph-mon[98773]: osdmap e134: 8 total, 7 up, 8 in 2024-04-10T03:23:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:20 smithi152 ceph-mon[88640]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-10T03:23:20.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:20 smithi152 ceph-mon[88640]: osdmap e134: 8 total, 7 up, 8 in 2024-04-10T03:23:20.932 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:20 smithi102 podman[130154]: 2024-04-10 03:23:20.665119887 +0000 UTC m=+0.654858398 container remove 6f7249bbf2b8fd3640be6b3686002a44362e26a106d339db9c5120a69280af48 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-10T03:23:21.202 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:20] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:23:21.203 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:20.977546903 +0000 UTC m=+0.018837783 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:21.203 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:21.082653792 +0000 UTC m=+0.123944672 container create 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2) 2024-04-10T03:23:21.457 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[94535]: osdmap e135: 8 total, 7 up, 8 in 2024-04-10T03:23:21.457 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[94535]: pgmap v488: 73 pgs: 10 peering, 9 stale+active+clean, 54 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:21.457 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/413316606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:21.457 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[98773]: osdmap e135: 8 total, 7 up, 8 in 2024-04-10T03:23:21.458 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[98773]: pgmap v488: 73 pgs: 10 peering, 9 stale+active+clean, 54 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:21.458 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:21 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/413316606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:21.748 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:21.460761567 +0000 UTC m=+0.502052447 container init 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-10T03:23:21.749 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:21.464154013 +0000 UTC m=+0.505444901 container start 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240403, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:23:21.749 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:21.560539769 +0000 UTC m=+0.601830651 container attach 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, ceph=True) 2024-04-10T03:23:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:21 smithi152 ceph-mon[88640]: osdmap e135: 8 total, 7 up, 8 in 2024-04-10T03:23:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:21 smithi152 ceph-mon[88640]: pgmap v488: 73 pgs: 10 peering, 9 stale+active+clean, 54 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:21.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:21 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/413316606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:22.036 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:21 smithi102 podman[130247]: 2024-04-10 03:23:21.75232012 +0000 UTC m=+0.793611009 container died 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, io.buildah.version=1.33.5) 2024-04-10T03:23:22.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:22 smithi152 ceph-mon[88640]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-10T03:23:22.775 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:22 smithi102 ceph-mon[94535]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-10T03:23:22.776 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:22 smithi102 ceph-mon[98773]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-10T03:23:22.776 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 podman[130298]: 2024-04-10 03:23:22.339949597 +0000 UTC m=+0.581114885 container remove 7de03b135101662ce564ef84085df3ee484a74850a89cad72345d0e248a64683 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:23:22.776 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.3.service: Deactivated successfully. 2024-04-10T03:23:22.776 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 systemd[1]: Stopped Ceph osd.3 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:23:22.776 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.3.service: Consumed 29min 49.847s CPU time. 2024-04-10T03:23:22.776 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 systemd[1]: Starting Ceph osd.3 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:23:23.041 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 podman[130402]: 2024-04-10 03:23:22.682177508 +0000 UTC m=+0.018855221 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:23.041 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:22 smithi102 podman[130402]: 2024-04-10 03:23:22.79505821 +0000 UTC m=+0.131735938 container create 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:23:23.355 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 podman[130402]: 2024-04-10 03:23:23.052650048 +0000 UTC m=+0.389327735 container init 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0) 2024-04-10T03:23:23.355 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 podman[130402]: 2024-04-10 03:23:23.055847382 +0000 UTC m=+0.392525071 container start 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-10T03:23:23.355 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 podman[130402]: 2024-04-10 03:23:23.152666592 +0000 UTC m=+0.489344289 container attach 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:23:23.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:23 smithi152 ceph-mon[88640]: pgmap v489: 73 pgs: 10 peering, 7 stale+active+clean, 4 active+undersized, 52 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:23.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-mon[94535]: pgmap v489: 73 pgs: 10 peering, 7 stale+active+clean, 4 active+undersized, 52 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:23.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-mon[98773]: pgmap v489: 73 pgs: 10 peering, 7 stale+active+clean, 4 active+undersized, 52 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:23.791 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-10T03:23:23.791 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-10T03:23:23.791 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 bash[130402]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate[130417]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-10T03:23:23.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:23 smithi102 podman[130402]: 2024-04-10 03:23:23.526491996 +0000 UTC m=+0.863169698 container died 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_CLEAN=True, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-10T03:23:24.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:24 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/176492478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:24.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:24 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/176492478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:24.541 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 podman[130573]: 2024-04-10 03:23:24.267855272 +0000 UTC m=+0.736588121 container remove 2ec1b85f6680ebae7d2d0a6242155ac22501dfb6ea57864bcdf980a2fa71c5a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:23:24.542 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 podman[130641]: 2024-04-10 03:23:24.397962226 +0000 UTC m=+0.018947646 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:24.542 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 podman[130641]: 2024-04-10 03:23:24.502156546 +0000 UTC m=+0.123141997 container create 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-10T03:23:24.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:24 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/176492478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:25.041 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 podman[130641]: 2024-04-10 03:23:24.70453044 +0000 UTC m=+0.325515849 container init 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:23:25.042 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 podman[130641]: 2024-04-10 03:23:24.707730548 +0000 UTC m=+0.328715966 container start 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:23:25.042 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 ceph-osd[130660]: -- 172.21.15.102:0/3797342443 <== mon.2 v2:172.21.15.102:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e5dac7b680 con 0x55e5dac45400 2024-04-10T03:23:25.042 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 bash[130641]: 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba 2024-04-10T03:23:25.042 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:24 smithi102 systemd[1]: Started Ceph osd.3 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:23:25.541 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:23:25.254+0000 7fd07364c7c0 -1 Falling back to public interface 2024-04-10T03:23:25.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[94535]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:25.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[98773]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:25 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:25 smithi152 ceph-mon[88640]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:25.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:25 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:26.699 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:26 smithi102 ceph-mon[94535]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:26.699 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1843944657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:26.699 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:26 smithi102 ceph-mon[98773]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:26.699 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1843944657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:26 smithi152 ceph-mon[88640]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:26.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1843944657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:27.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:27 smithi152 ceph-mon[88640]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:27.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:27 smithi102 ceph-mon[94535]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:27.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:27 smithi102 ceph-mon[98773]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:28.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:28 smithi102 ceph-mon[94535]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-10T03:23:28.735 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1347804753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:28.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:28 smithi102 ceph-mon[98773]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-10T03:23:28.736 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1347804753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:28 smithi152 ceph-mon[88640]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-10T03:23:28.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1347804753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[98773]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:29.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:29 smithi102 ceph-mon[94535]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:29 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:29 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:29 smithi152 ceph-mon[88640]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:30.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1160397022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:30.935 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1160397022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:30 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:31.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:30 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1160397022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:31.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:30] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:23:31.291 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:31 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:23:31.055+0000 7fd07364c7c0 -1 osd.3 133 log_to_monitors true 2024-04-10T03:23:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:31 smithi152 ceph-mon[88640]: pgmap v493: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:32.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:31 smithi152 ceph-mon[88640]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:23:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:31 smithi102 ceph-mon[94535]: pgmap v493: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:31 smithi102 ceph-mon[94535]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:23:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:31 smithi102 ceph-mon[98773]: pgmap v493: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:32.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:31 smithi102 ceph-mon[98773]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-10T03:23:32.507 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:23:32.131+0000 7fd060a9c700 -1 osd.3 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-10T03:23:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[94535]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:23:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[94535]: osdmap e136: 8 total, 7 up, 8 in 2024-04-10T03:23:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[94535]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:23:32.791 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[94535]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' 2024-04-10T03:23:32.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[98773]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:23:32.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[98773]: osdmap e136: 8 total, 7 up, 8 in 2024-04-10T03:23:32.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[98773]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:23:32.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:32 smithi102 ceph-mon[98773]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' 2024-04-10T03:23:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:32 smithi152 ceph-mon[88640]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-10T03:23:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:32 smithi152 ceph-mon[88640]: osdmap e136: 8 total, 7 up, 8 in 2024-04-10T03:23:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:32 smithi152 ceph-mon[88640]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-04-10T03:23:33.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:32 smithi152 ceph-mon[88640]: from='osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076]' entity='osd.3' 2024-04-10T03:23:33.686 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:23:33.507+0000 7f1b6ff70700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-10T03:23:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: pgmap v495: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:34.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2511835302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076] boot 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: osdmap e137: 8 total, 8 up, 8 in 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: pgmap v495: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2511835302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076] boot 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: osdmap e137: 8 total, 8 up, 8 in 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: pgmap v495: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2511835302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: osd.3 [v2:172.21.15.102:6826/2229527076,v1:172.21.15.102:6827/2229527076] boot 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: osdmap e137: 8 total, 8 up, 8 in 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:34.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:34.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:35.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:34 smithi152 ceph-mon[88640]: pgmap v497: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:34 smithi152 ceph-mon[88640]: pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:34 smithi152 ceph-mon[88640]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:35.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:34 smithi152 ceph-mon[88640]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-10T03:23:35.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:34 smithi152 ceph-mon[88640]: osdmap e138: 8 total, 8 up, 8 in 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[98773]: pgmap v497: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[98773]: pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[98773]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[98773]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[98773]: osdmap e138: 8 total, 8 up, 8 in 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[94535]: pgmap v497: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[94535]: pgmap v498: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-10T03:23:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[94535]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-10T03:23:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[94535]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-10T03:23:35.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:34 smithi102 ceph-mon[94535]: osdmap e138: 8 total, 8 up, 8 in 2024-04-10T03:23:35.245 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 4 2024-04-10T03:23:35.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:35 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2799005072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:35.982 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:35 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2799005072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:36.008 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch upgrade pause' 2024-04-10T03:23:36.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:35 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2799005072' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-10T03:23:36.271 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:36 smithi152 ceph-mon[88640]: pgmap v500: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 53 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:37.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:36 smithi152 ceph-mon[88640]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:37.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:36 smithi102 ceph-mon[94535]: pgmap v500: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 53 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:37.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:36 smithi102 ceph-mon[94535]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:37.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:36 smithi102 ceph-mon[98773]: pgmap v500: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 53 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-10T03:23:37.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:36 smithi102 ceph-mon[98773]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2024-04-10T03:23:37.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:37 smithi102 ceph-mon[94535]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-10T03:23:37.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:37 smithi102 ceph-mon[98773]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-10T03:23:38.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:37 smithi152 ceph-mon[88640]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2024-04-10T03:23:38.198 INFO:teuthology.orchestra.run.smithi102.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:38.830 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:38 smithi102 ceph-mon[98773]: pgmap v501: 73 pgs: 5 active+undersized, 68 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:38.830 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:38 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:38.831 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:38 smithi102 ceph-mon[94535]: pgmap v501: 73 pgs: 5 active+undersized, 68 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:38.831 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:38 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:38.857 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:23:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:38 smithi152 ceph-mon[88640]: pgmap v501: 73 pgs: 5 active+undersized, 68 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:39.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:38 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:39.114 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:39 smithi152 ceph-mon[88640]: from='client.35289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:40.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:39 smithi152 ceph-mon[88640]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:40.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:39 smithi102 ceph-mon[94535]: from='client.35289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:40.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:39 smithi102 ceph-mon[94535]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:39 smithi102 ceph-mon[98773]: from='client.35289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:40.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:39 smithi102 ceph-mon[98773]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:23:40.964 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:23:40.964 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (16m) 12s ago 20m 23.8M - 0.25.0 c8568f914cd2 65832d2b0608 2024-04-10T03:23:40.964 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 error 72s ago 20m - - 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (19m) 12s ago 19m 19.5M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (19m) 72s ago 19m 17.9M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443,9283,8765 running (14m) 72s ago 24m 455M - 19.0.0-2051-g3a866975 06a486fd9bae 10f3bdd94f98 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:8443,9283,8765 running (15m) 12s ago 26m 512M - 19.0.0-2051-g3a866975 06a486fd9bae ba38a088e643 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (14m) 12s ago 26m 67.4M 2048M 19.0.0-2051-g3a866975 06a486fd9bae c3949a498743 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 running (10m) 72s ago 25m 50.0M 2048M 19.0.0-2051-g3a866975 06a486fd9bae 8c2b763e28e9 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 running (12m) 12s ago 25m 50.0M 2048M 19.0.0-2051-g3a866975 06a486fd9bae ff269d657b5a 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (16m) 12s ago 20m 15.9M - 1.5.0 0da6a335fe13 773a048f8daf 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (15m) 72s ago 20m 16.5M - 1.5.0 0da6a335fe13 6010c1e76260 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (8m) 12s ago 24m 572M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 0188a8a40b84 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (4m) 12s ago 24m 343M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 2b12e2e39488 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (51s) 12s ago 23m 79.0M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 6784651177eb 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (16s) 12s ago 23m 12.3M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 7d8b0c0335ac 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (22m) 72s ago 22m 580M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (22m) 72s ago 22m 898M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:23:40.965 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (22m) 72s ago 22m 827M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:23:40.966 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (21m) 72s ago 21m 697M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:23:40.966 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (14m) 72s ago 21m 65.5M - 2.43.0 a07b618ecd1d a7900639fc5e 2024-04-10T03:23:40.977 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:40 smithi102 ceph-mon[94535]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:40.977 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:40 smithi102 ceph-mon[98773]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:40.977 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:40] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.43.0" 2024-04-10T03:23:41.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:40 smithi152 ceph-mon[88640]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:41.639 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph orch ps' 2024-04-10T03:23:41.914 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:42.899 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:42 smithi102 ceph-mon[94535]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:42.900 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:42 smithi102 ceph-mon[94535]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:42.900 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:42 smithi102 ceph-mon[98773]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:42.900 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:42 smithi102 ceph-mon[98773]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:42 smithi152 ceph-mon[88640]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:43.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:42 smithi152 ceph-mon[88640]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:43.797 INFO:teuthology.orchestra.run.smithi102.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:alertmanager.a smithi102 *:9093,9094 running (16m) 15s ago 20m 23.8M - 0.25.0 c8568f914cd2 65832d2b0608 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:grafana.a smithi152 *:3000 error 75s ago 20m - - 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi102.exkvif smithi102 running (19m) 15s ago 19m 19.5M - 17.2.7-827-gd3b5060a 21de9b7369ad 3a69e9f39ba4 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mds.foo.smithi152.adssru smithi152 running (19m) 75s ago 19m 17.9M - 17.2.7-827-gd3b5060a 21de9b7369ad cc039688f6f8 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.x smithi152 *:8443,9283,8765 running (14m) 75s ago 24m 455M - 19.0.0-2051-g3a866975 06a486fd9bae 10f3bdd94f98 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mgr.y smithi102 *:8443,9283,8765 running (15m) 15s ago 26m 512M - 19.0.0-2051-g3a866975 06a486fd9bae ba38a088e643 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mon.a smithi102 running (14m) 15s ago 26m 67.4M 2048M 19.0.0-2051-g3a866975 06a486fd9bae c3949a498743 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mon.b smithi152 running (10m) 75s ago 25m 50.0M 2048M 19.0.0-2051-g3a866975 06a486fd9bae 8c2b763e28e9 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:mon.c smithi102 running (12m) 15s ago 25m 50.0M 2048M 19.0.0-2051-g3a866975 06a486fd9bae ff269d657b5a 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.a smithi102 *:9100 running (16m) 15s ago 20m 15.9M - 1.5.0 0da6a335fe13 773a048f8daf 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:node-exporter.b smithi152 *:9100 running (15m) 75s ago 20m 16.5M - 1.5.0 0da6a335fe13 6010c1e76260 2024-04-10T03:23:43.798 INFO:teuthology.orchestra.run.smithi102.stdout:osd.0 smithi102 running (9m) 15s ago 24m 572M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 0188a8a40b84 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.1 smithi102 running (4m) 15s ago 24m 343M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 2b12e2e39488 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.2 smithi102 running (54s) 15s ago 23m 79.0M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 6784651177eb 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.3 smithi102 running (19s) 15s ago 23m 12.3M 2480M 19.0.0-2051-g3a866975 06a486fd9bae 7d8b0c0335ac 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.4 smithi152 running (22m) 75s ago 22m 580M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 20decec246f6 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.5 smithi152 running (22m) 75s ago 22m 898M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad 7eb970dbdd2d 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.6 smithi152 running (22m) 75s ago 22m 827M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d97f417e008b 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:osd.7 smithi152 running (21m) 75s ago 21m 697M 2480M 17.2.7-827-gd3b5060a 21de9b7369ad d60bd93ae519 2024-04-10T03:23:43.799 INFO:teuthology.orchestra.run.smithi102.stdout:prometheus.a smithi152 *:9095 running (14m) 75s ago 21m 65.5M - 2.43.0 a07b618ecd1d a7900639fc5e 2024-04-10T03:23:44.540 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -e sha1=3a866975a60f6dc859246f6f19fd8ddb4db2cc2e -- bash -c 'ceph versions' 2024-04-10T03:23:44.761 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:44 smithi102 ceph-mon[94535]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:44.761 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:44 smithi102 ceph-mon[94535]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:44.761 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:44 smithi102 ceph-mon[98773]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:44.761 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:44 smithi102 ceph-mon[98773]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:44.799 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:44 smithi152 ceph-mon[88640]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:45.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:44 smithi152 ceph-mon[88640]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-10T03:23:46.750 INFO:teuthology.orchestra.run.smithi102.stdout:{ 2024-04-10T03:23:46.750 INFO:teuthology.orchestra.run.smithi102.stdout: "mon": { 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 3 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: }, 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "mgr": { 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 2 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: }, 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "osd": { 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 4, 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 4 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: }, 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "mds": { 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 2 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: }, 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "overall": { 2024-04-10T03:23:46.751 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 17.2.7-827-gd3b5060a (d3b5060a17449edfae9385cdf792afec109fca5d) quincy (stable)": 6, 2024-04-10T03:23:46.752 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph version 19.0.0-2051-g3a866975 (3a866975a60f6dc859246f6f19fd8ddb4db2cc2e) squid (dev)": 9 2024-04-10T03:23:46.752 INFO:teuthology.orchestra.run.smithi102.stdout: } 2024-04-10T03:23:46.752 INFO:teuthology.orchestra.run.smithi102.stdout:} 2024-04-10T03:23:47.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:46 smithi102 ceph-mon[94535]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:47.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3191922446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:46 smithi102 ceph-mon[98773]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:47.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3191922446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:47.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:46 smithi152 ceph-mon[88640]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:47.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3191922446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-10T03:23:47.528 DEBUG:teuthology.parallel:result is None 2024-04-10T03:23:47.528 INFO:teuthology.run_tasks:Running task print... 2024-04-10T03:23:47.538 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-10T03:23:47.538 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-10T03:23:47.557 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-10T03:23:47.557 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-10T03:23:47.557 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-10T03:23:47.820 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[94535]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[98773]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:49.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:48 smithi152 ceph-mon[88640]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:49.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:23:49.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:49.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:49.978 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:23:50.644 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":40,"flags":0,"active_gid":25253,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":2068836803},{"type":"v1","addr":"172.21.15.102:6801","nonce":2068836803}]},"active_addr":"172.21.15.102:6801/2068836803","active_change":"2024-04-10T03:09:32.746044+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25060,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.102:8443/","prometheus":"http://172.21.15.102:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":81,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":3979899912}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1068960327}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":804428500}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.102:0","nonce":1085276194}]}]} 2024-04-10T03:23:50.649 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-10T03:23:50.649 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-10T03:23:50.649 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:23:50.906 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:50.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:50 smithi102 ceph-mon[94535]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:50.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:50 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2923007599' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:23:50.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:50 smithi102 ceph-mon[98773]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:50.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:50 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2923007599' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:23:51.057 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:50 smithi152 ceph-mon[88640]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:51.057 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:50 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2923007599' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-10T03:23:51.246 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:23:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:23:50] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:23:52.610 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:23:52.610 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":138,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:23:34.131376+0000","last_up_change":"2024-04-10T03:23:33.127068+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-10T03:03:50.907839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-10T03:03:51.409886+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":137,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6803","nonce":2888888743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6805","nonce":2888888743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6809","nonce":2888888743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6807","nonce":2888888743}]},"public_addr":"172.21.15.102:6803/2888888743","cluster_addr":"172.21.15.102:6805/2888888743","heartbeat_back_addr":"172.21.15.102:6809/2888888743","heartbeat_front_addr":"172.21.15.102:6807/2888888743","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":110,"up_from":114,"up_thru":137,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6811","nonce":3348168242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6813","nonce":3348168242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6817","nonce":3348168242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6815","nonce":3348168242}]},"public_addr":"172.21.15.102:6811/3348168242","cluster_addr":"172.21.15.102:6813/3348168242","heartbeat_back_addr":"172.21.15.102:6817/3348168242","heartbeat_front_addr":"172.21.15.102:6815/3348168242","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":128,"up_from":132,"up_thru":137,"down_at":129,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6819","nonce":4061930862}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6821","nonce":4061930862}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6825","nonce":4061930862}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6823","nonce":4061930862}]},"public_addr":"172.21.15.102:6819/4061930862","cluster_addr":"172.21.15.102:6821/4061930862","heartbeat_back_addr":"172.21.15.102:6825/4061930862","heartbeat_front_addr":"172.21.15.102:6823/4061930862","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":133,"up_from":137,"up_thru":137,"down_at":134,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6827","nonce":2229527076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6829","nonce":2229527076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6833","nonce":2229527076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6831","nonce":2229527076}]},"public_addr":"172.21.15.102:6827/2229527076","cluster_addr":"172.21.15.102:6829/2229527076","heartbeat_back_addr":"172.21.15.102:6833/2229527076","heartbeat_front_addr":"172.21.15.102:6831/2229527076","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":134,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":110},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":128},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":133},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/3329839571":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2886930243":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.152:0/1852145602":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.152:0/2960675918":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/2292901015":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6800/3562317133":"2024-04-11T03:06:58.484004+0000","172.21.15.152:6835/2088203104":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6801/3562317133":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.152:0/1895702089":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.152:0/1419948851":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/1827401448":"2024-04-11T03:06:58.484004+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.152:6834/2088203104":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2965564602":"2024-04-11T03:06:58.484004+0000","172.21.15.152:0/4241207468":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/3082620748":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/1928572397":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.152:0/683315366":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/1592407319":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2562808467":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2533379491":"2024-04-11T03:09:32.745818+0000","172.21.15.102:6801/4270369458":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/4270369458":"2024-04-11T03:09:32.745818+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:23:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[94535]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2252751037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:53.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[98773]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:53.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2252751037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:53.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:52 smithi152 ceph-mon[88640]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:53.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:52 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:52 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:53.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2252751037' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:53.350 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-10T03:23:53.350 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-10T03:23:53.604 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:54.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:54.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:54.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:23:54.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:23:54.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:23:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[98773]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[98773]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[98773]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[98773]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[94535]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[94535]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[94535]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:54 smithi102 ceph-mon[94535]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:54 smithi152 ceph-mon[88640]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:54 smithi152 ceph-mon[88640]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:54 smithi152 ceph-mon[88640]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:54 smithi152 ceph-mon[88640]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:55.555 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:23:55.555 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":138,"fsid":"e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4","created":"2024-04-10T02:57:00.532773+0000","modified":"2024-04-10T03:23:34.131376+0000","last_up_change":"2024-04-10T03:23:33.127068+0000","last_in_change":"2024-04-10T03:01:46.806772+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-10T03:00:04.753220+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-10T03:02:36.898804+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-10T03:03:50.907839+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-10T03:03:51.409886+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"189ba4fa-2bb5-4bd0-a97b-bc549c5a9311","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":137,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6803","nonce":2888888743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6805","nonce":2888888743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6809","nonce":2888888743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":2888888743},{"type":"v1","addr":"172.21.15.102:6807","nonce":2888888743}]},"public_addr":"172.21.15.102:6803/2888888743","cluster_addr":"172.21.15.102:6805/2888888743","heartbeat_back_addr":"172.21.15.102:6809/2888888743","heartbeat_front_addr":"172.21.15.102:6807/2888888743","state":["exists","up"]},{"osd":1,"uuid":"b295353c-a779-4ced-917a-64e180c8b379","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":110,"up_from":114,"up_thru":137,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6811","nonce":3348168242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6813","nonce":3348168242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6817","nonce":3348168242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":3348168242},{"type":"v1","addr":"172.21.15.102:6815","nonce":3348168242}]},"public_addr":"172.21.15.102:6811/3348168242","cluster_addr":"172.21.15.102:6813/3348168242","heartbeat_back_addr":"172.21.15.102:6817/3348168242","heartbeat_front_addr":"172.21.15.102:6815/3348168242","state":["exists","up"]},{"osd":2,"uuid":"3601bd6b-a4e3-41ba-8688-15dcc451f5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":128,"up_from":132,"up_thru":137,"down_at":129,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6819","nonce":4061930862}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6821","nonce":4061930862}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6825","nonce":4061930862}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":4061930862},{"type":"v1","addr":"172.21.15.102:6823","nonce":4061930862}]},"public_addr":"172.21.15.102:6819/4061930862","cluster_addr":"172.21.15.102:6821/4061930862","heartbeat_back_addr":"172.21.15.102:6825/4061930862","heartbeat_front_addr":"172.21.15.102:6823/4061930862","state":["exists","up"]},{"osd":3,"uuid":"152f506d-f4fc-44d1-8252-0b994b129f90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":133,"up_from":137,"up_thru":137,"down_at":134,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6827","nonce":2229527076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6829","nonce":2229527076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6832","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6833","nonce":2229527076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":2229527076},{"type":"v1","addr":"172.21.15.102:6831","nonce":2229527076}]},"public_addr":"172.21.15.102:6827/2229527076","cluster_addr":"172.21.15.102:6829/2229527076","heartbeat_back_addr":"172.21.15.102:6833/2229527076","heartbeat_front_addr":"172.21.15.102:6831/2229527076","state":["exists","up"]},{"osd":4,"uuid":"382c51f3-c988-48aa-9f0d-3ad491ad17cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6801","nonce":1841668292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6803","nonce":1841668292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6807","nonce":1841668292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":1841668292},{"type":"v1","addr":"172.21.15.152:6805","nonce":1841668292}]},"public_addr":"172.21.15.152:6801/1841668292","cluster_addr":"172.21.15.152:6803/1841668292","heartbeat_back_addr":"172.21.15.152:6807/1841668292","heartbeat_front_addr":"172.21.15.152:6805/1841668292","state":["exists","up"]},{"osd":5,"uuid":"061a33e8-7b37-4e6b-b1bb-9abd4b5bc7ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6809","nonce":537038863}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6811","nonce":537038863}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6815","nonce":537038863}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":537038863},{"type":"v1","addr":"172.21.15.152:6813","nonce":537038863}]},"public_addr":"172.21.15.152:6809/537038863","cluster_addr":"172.21.15.152:6811/537038863","heartbeat_back_addr":"172.21.15.152:6815/537038863","heartbeat_front_addr":"172.21.15.152:6813/537038863","state":["exists","up"]},{"osd":6,"uuid":"64f5afa7-c8f8-448d-a0c7-e220655282db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":134,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6817","nonce":524183471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6819","nonce":524183471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6823","nonce":524183471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":524183471},{"type":"v1","addr":"172.21.15.152:6821","nonce":524183471}]},"public_addr":"172.21.15.152:6817/524183471","cluster_addr":"172.21.15.152:6819/524183471","heartbeat_back_addr":"172.21.15.152:6823/524183471","heartbeat_front_addr":"172.21.15.152:6821/524183471","state":["exists","up"]},{"osd":7,"uuid":"c0d07434-ccb1-4d71-92aa-bda52e0b6edb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":137,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6825","nonce":2303235160}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6827","nonce":2303235160}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6831","nonce":2303235160}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":2303235160},{"type":"v1","addr":"172.21.15.152:6829","nonce":2303235160}]},"public_addr":"172.21.15.152:6825/2303235160","cluster_addr":"172.21.15.152:6827/2303235160","heartbeat_back_addr":"172.21.15.152:6831/2303235160","heartbeat_front_addr":"172.21.15.152:6829/2303235160","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:13.438103+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T02:59:37.337817+0000","dead_epoch":110},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:01.711301+0000","dead_epoch":128},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:27.019914+0000","dead_epoch":133},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:00:49.342843+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:13.409089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:01:37.641324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-10T03:02:02.624739+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.102:0/3329839571":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2886930243":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2968291715":"2024-04-11T03:02:36.065876+0000","172.21.15.152:0/1852145602":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/426289621":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/518509927":"2024-04-11T02:57:54.437842+0000","172.21.15.152:0/2960675918":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/2508804905":"2024-04-11T02:57:54.437842+0000","172.21.15.102:0/2292901015":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6800/3562317133":"2024-04-11T03:06:58.484004+0000","172.21.15.152:6835/2088203104":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6801/3562317133":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6800/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.152:0/1895702089":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6800/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.152:0/1419948851":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/2920496069":"2024-04-11T02:57:29.929627+0000","172.21.15.102:0/1827401448":"2024-04-11T03:06:58.484004+0000","172.21.15.102:0/4255250819":"2024-04-11T02:57:29.929627+0000","172.21.15.102:6801/3793380741":"2024-04-11T02:57:54.437842+0000","172.21.15.152:6834/2088203104":"2024-04-11T03:08:25.773150+0000","172.21.15.102:6801/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/3041747516":"2024-04-11T03:02:36.065876+0000","172.21.15.102:6800/2812746913":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2475710492":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/2965564602":"2024-04-11T03:06:58.484004+0000","172.21.15.152:0/4241207468":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/3082620748":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/18848219":"2024-04-11T03:02:36.065876+0000","172.21.15.102:0/1928572397":"2024-04-11T03:06:58.484004+0000","172.21.15.102:6801/3051857918":"2024-04-11T02:57:29.929627+0000","172.21.15.152:0/683315366":"2024-04-11T03:08:25.773150+0000","172.21.15.102:0/1592407319":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2562808467":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/2533379491":"2024-04-11T03:09:32.745818+0000","172.21.15.102:6801/4270369458":"2024-04-11T03:09:32.745818+0000","172.21.15.102:0/528648107":"2024-04-11T02:57:54.437842+0000","172.21.15.102:6800/4270369458":"2024-04-11T03:09:32.745818+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-10T03:23:56.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:55 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/212909389' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:56.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:55 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/212909389' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:56.200 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-10T03:23:56.200 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-10T03:23:56.201 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-10T03:23:56.201 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-10T03:23:56.201 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-10T03:23:56.202 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-10T03:23:56.202 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-10T03:23:56.202 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-10T03:23:56.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:55 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/212909389' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-10T03:23:56.701 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.723 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.783 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.784 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.784 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.817 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:56.819 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:23:57.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:56 smithi102 ceph-mon[94535]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:57.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:56 smithi102 ceph-mon[98773]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:57.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:56 smithi152 ceph-mon[88640]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:59.010 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:23:58 smithi102 ceph-mon[94535]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:59.010 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:23:58 smithi102 ceph-mon[98773]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:23:59.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:23:58 smithi152 ceph-mon[88640]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:01.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:00 smithi102 ceph-mon[94535]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:01.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:00 smithi102 ceph-mon[98773]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:01.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:00] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:24:01.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:00 smithi152 ceph-mon[88640]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:02.085 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:03.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:02 smithi102 ceph-mon[94535]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:03.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:02 smithi102 ceph-mon[98773]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:03.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:02 smithi152 ceph-mon[88640]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:04.044 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:04.044 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:04.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:05.078 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:04 smithi102 ceph-mon[94535]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:05.079 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:04 smithi102 ceph-mon[98773]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:05.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:04 smithi152 ceph-mon[88640]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:06.924 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:06 smithi102 ceph-mon[94535]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:06.925 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:06 smithi102 ceph-mon[98773]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:07.063 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:06 smithi152 ceph-mon[88640]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:08.879 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:08 smithi102 ceph-mon[94535]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:08.880 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:08 smithi102 ceph-mon[98773]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:09.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:08 smithi152 ceph-mon[88640]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:10.039 INFO:teuthology.orchestra.run.smithi102.stdout:197568495889 2024-04-10T03:24:10.040 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-10T03:24:10.694 INFO:teuthology.orchestra.run.smithi102.stdout:566935683088 2024-04-10T03:24:10.694 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-10T03:24:10.917 INFO:teuthology.orchestra.run.smithi102.stdout:231928234251 2024-04-10T03:24:10.917 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-10T03:24:10.933 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:10 smithi102 ceph-mon[94535]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:10.933 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:10 smithi102 ceph-mon[98773]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:11.240 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:10] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:24:11.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:10 smithi152 ceph-mon[88640]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:11.693 INFO:teuthology.orchestra.run.smithi102.stdout:489626271806 2024-04-10T03:24:11.693 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-10T03:24:12.091 INFO:teuthology.orchestra.run.smithi102.stdout:163208757524 2024-04-10T03:24:12.091 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-10T03:24:12.741 INFO:teuthology.orchestra.run.smithi102.stdout:128849019163 2024-04-10T03:24:12.741 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-10T03:24:12.971 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:12 smithi102 ceph-mon[94535]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:12.972 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:12 smithi102 ceph-mon[98773]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:13.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:12 smithi152 ceph-mon[88640]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:13.407 INFO:teuthology.orchestra.run.smithi102.stdout:588410519561 2024-04-10T03:24:13.408 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-10T03:24:13.926 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:13.926 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.395 INFO:teuthology.orchestra.run.smithi102.stdout:390842024050 2024-04-10T03:24:14.396 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-10T03:24:14.409 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.409 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.738 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.738 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.857 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:14.940 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:14 smithi102 ceph-mon[94535]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:14.940 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:14 smithi102 ceph-mon[98773]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:15.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:14 smithi152 ceph-mon[88640]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:17.034 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:16 smithi102 ceph-mon[94535]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:17.034 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:16 smithi102 ceph-mon[98773]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:17.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:16 smithi152 ceph-mon[88640]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:19.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:18 smithi102 ceph-mon[94535]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:19.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:19.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:18 smithi102 ceph-mon[98773]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:19.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:19.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:18 smithi152 ceph-mon[88640]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:19.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:21.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:20 smithi152 ceph-mon[88640]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:21.281 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:20 smithi102 ceph-mon[94535]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:21.282 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:20 smithi102 ceph-mon[98773]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:21.282 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:20] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:24:21.734 INFO:teuthology.orchestra.run.smithi102.stdout:197568495891 2024-04-10T03:24:21.907 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:21 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/646698600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:24:21.907 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:21 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/646698600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:24:22.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:21 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/646698600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-10T03:24:22.512 INFO:teuthology.orchestra.run.smithi102.stdout:588410519563 2024-04-10T03:24:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:22 smithi102 ceph-mon[94535]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:23.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3192143185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:24:23.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:22 smithi102 ceph-mon[98773]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:23.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3192143185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:24:23.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:22 smithi152 ceph-mon[88640]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:23.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3192143185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-10T03:24:23.858 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495889 got 197568495891 for osd.6 2024-04-10T03:24:23.858 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:24.153 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:23 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2860721898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:24:24.153 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:23 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2860721898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:24:24.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:23 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2860721898' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-10T03:24:24.471 INFO:tasks.cephadm.ceph_manager.ceph:need seq 588410519561 got 588410519563 for osd.3 2024-04-10T03:24:24.471 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:24.919 INFO:teuthology.orchestra.run.smithi102.stdout:163208757527 2024-04-10T03:24:25.093 INFO:teuthology.orchestra.run.smithi102.stdout:231928234254 2024-04-10T03:24:25.093 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:24 smithi102 ceph-mon[94535]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:25.093 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:24 smithi102 ceph-mon[98773]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:25.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:24 smithi152 ceph-mon[88640]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:25.698 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234251 got 231928234254 for osd.7 2024-04-10T03:24:25.699 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:25.883 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:25 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/472963842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:24:25.883 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:25 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/472963842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:24:26.187 INFO:teuthology.orchestra.run.smithi102.stdout:128849019166 2024-04-10T03:24:26.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:25 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/472963842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-10T03:24:26.890 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:26 smithi102 ceph-mon[94535]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:26.890 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/665135337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:24:26.890 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:26 smithi102 ceph-mon[98773]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:26.891 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/665135337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:24:26.919 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019163 got 128849019166 for osd.4 2024-04-10T03:24:26.919 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:27.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:26 smithi152 ceph-mon[88640]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:27.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/665135337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-10T03:24:27.857 INFO:teuthology.orchestra.run.smithi102.stdout:566935683092 2024-04-10T03:24:28.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:27 smithi102 ceph-mon[98773]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:27 smithi102 ceph-mon[94535]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:28.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:27 smithi152 ceph-mon[88640]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:28.496 INFO:teuthology.orchestra.run.smithi102.stdout:489626271810 2024-04-10T03:24:28.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 566935683088 got 566935683092 for osd.2 2024-04-10T03:24:28.724 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:29.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/649852253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:24:29.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/225003615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:24:29.022 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/649852253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:24:29.022 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/225003615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:24:29.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/649852253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-10T03:24:29.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/225003615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-10T03:24:29.339 INFO:tasks.cephadm.ceph_manager.ceph:need seq 489626271806 got 489626271810 for osd.1 2024-04-10T03:24:29.340 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:29.373 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:29.761 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757524 got 163208757527 for osd.5 2024-04-10T03:24:29.761 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:30.035 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:29 smithi102 ceph-mon[94535]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:30.035 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:29 smithi102 ceph-mon[98773]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:30.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:29 smithi152 ceph-mon[88640]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:31.260 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:30] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:24:31.679 INFO:teuthology.orchestra.run.smithi102.stdout:390842024053 2024-04-10T03:24:32.448 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:32 smithi102 ceph-mon[94535]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:32.448 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:32 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/864338554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:24:32.448 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:32 smithi102 ceph-mon[98773]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:32.449 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:32 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/864338554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:24:32.476 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024050 got 390842024053 for osd.0 2024-04-10T03:24:32.476 DEBUG:teuthology.parallel:result is None 2024-04-10T03:24:32.476 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-10T03:24:32.476 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:24:32.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:32 smithi152 ceph-mon[88640]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:32.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:32 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/864338554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-10T03:24:32.734 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:33.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:33.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:33.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:34.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:34 smithi152 ceph-mon[88640]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:34.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:34 smithi102 ceph-mon[94535]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:34.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:34 smithi102 ceph-mon[98773]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:34.637 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:34.639 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:24:35.257 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:35 smithi102 ceph-mon[98773]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:35.257 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:35 smithi102 ceph-mon[94535]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:35.281 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":532,"stamp":"2024-04-10T03:24:33.099443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3005620,"kb_used_data":14524,"kb_used_omap":29,"kb_used_meta":2990946,"kb_avail":746955596,"statfs":{"total":767960285184,"available":764882530304,"internally_reserved":0,"allocated":14872576,"data_stored":8908078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30056,"internal_metadata":3062729368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002921"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134440+0000","last_change":"2024-04-10T03:22:58.820683+0000","last_active":"2024-04-10T03:23:33.134440+0000","last_peered":"2024-04-10T03:23:33.134440+0000","last_clean":"2024-04-10T03:23:33.134440+0000","last_became_active":"2024-04-10T03:22:58.820333+0000","last_became_peered":"2024-04-10T03:22:58.820333+0000","last_unstale":"2024-04-10T03:23:33.134440+0000","last_undegraded":"2024-04-10T03:23:33.134440+0000","last_fullsized":"2024-04-10T03:23:33.134440+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:18:40.834614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.149503+0000","last_change":"2024-04-10T03:23:34.149503+0000","last_active":"2024-04-10T03:23:34.149503+0000","last_peered":"2024-04-10T03:23:34.149503+0000","last_clean":"2024-04-10T03:23:34.149503+0000","last_became_active":"2024-04-10T03:23:34.148752+0000","last_became_peered":"2024-04-10T03:23:34.148752+0000","last_unstale":"2024-04-10T03:23:34.149503+0000","last_undegraded":"2024-04-10T03:23:34.149503+0000","last_fullsized":"2024-04-10T03:23:34.149503+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:11:49.117446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151460+0000","last_change":"2024-04-10T03:23:34.151048+0000","last_active":"2024-04-10T03:23:34.151460+0000","last_peered":"2024-04-10T03:23:34.151460+0000","last_clean":"2024-04-10T03:23:34.151460+0000","last_became_active":"2024-04-10T03:23:34.150595+0000","last_became_peered":"2024-04-10T03:23:34.150595+0000","last_unstale":"2024-04-10T03:23:34.151460+0000","last_undegraded":"2024-04-10T03:23:34.151460+0000","last_fullsized":"2024-04-10T03:23:34.151460+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:24:48.196187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151087+0000","last_change":"2024-04-10T03:23:34.151087+0000","last_active":"2024-04-10T03:23:34.151087+0000","last_peered":"2024-04-10T03:23:34.151087+0000","last_clean":"2024-04-10T03:23:34.151087+0000","last_became_active":"2024-04-10T03:23:34.150662+0000","last_became_peered":"2024-04-10T03:23:34.150662+0000","last_unstale":"2024-04-10T03:23:34.151087+0000","last_undegraded":"2024-04-10T03:23:34.151087+0000","last_fullsized":"2024-04-10T03:23:34.151087+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:43:05.878112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156674+0000","last_change":"2024-04-10T03:23:34.156674+0000","last_active":"2024-04-10T03:23:34.156674+0000","last_peered":"2024-04-10T03:23:34.156674+0000","last_clean":"2024-04-10T03:23:34.156674+0000","last_became_active":"2024-04-10T03:23:34.155850+0000","last_became_peered":"2024-04-10T03:23:34.155850+0000","last_unstale":"2024-04-10T03:23:34.156674+0000","last_undegraded":"2024-04-10T03:23:34.156674+0000","last_fullsized":"2024-04-10T03:23:34.156674+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:20:28.375709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143505+0000","last_change":"2024-04-10T03:19:06.500857+0000","last_active":"2024-04-10T03:23:33.143505+0000","last_peered":"2024-04-10T03:23:33.143505+0000","last_clean":"2024-04-10T03:23:33.143505+0000","last_became_active":"2024-04-10T03:19:06.500122+0000","last_became_peered":"2024-04-10T03:19:06.500122+0000","last_unstale":"2024-04-10T03:23:33.143505+0000","last_undegraded":"2024-04-10T03:23:33.143505+0000","last_fullsized":"2024-04-10T03:23:33.143505+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:11:18.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153369+0000","last_change":"2024-04-10T03:23:34.153056+0000","last_active":"2024-04-10T03:23:34.153369+0000","last_peered":"2024-04-10T03:23:34.153369+0000","last_clean":"2024-04-10T03:23:34.153369+0000","last_became_active":"2024-04-10T03:23:34.152704+0000","last_became_peered":"2024-04-10T03:23:34.152704+0000","last_unstale":"2024-04-10T03:23:34.153369+0000","last_undegraded":"2024-04-10T03:23:34.153369+0000","last_fullsized":"2024-04-10T03:23:34.153369+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:53:15.431644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156190+0000","last_change":"2024-04-10T03:23:34.156190+0000","last_active":"2024-04-10T03:23:34.156190+0000","last_peered":"2024-04-10T03:23:34.156190+0000","last_clean":"2024-04-10T03:23:34.156190+0000","last_became_active":"2024-04-10T03:23:34.155393+0000","last_became_peered":"2024-04-10T03:23:34.155393+0000","last_unstale":"2024-04-10T03:23:34.156190+0000","last_undegraded":"2024-04-10T03:23:34.156190+0000","last_fullsized":"2024-04-10T03:23:34.156190+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:29:53.054637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134831+0000","last_change":"2024-04-10T03:19:06.509704+0000","last_active":"2024-04-10T03:23:33.134831+0000","last_peered":"2024-04-10T03:23:33.134831+0000","last_clean":"2024-04-10T03:23:33.134831+0000","last_became_active":"2024-04-10T03:19:06.509393+0000","last_became_peered":"2024-04-10T03:19:06.509393+0000","last_unstale":"2024-04-10T03:23:33.134831+0000","last_undegraded":"2024-04-10T03:23:33.134831+0000","last_fullsized":"2024-04-10T03:23:33.134831+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:20:15.251509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155786+0000","last_change":"2024-04-10T03:23:34.155786+0000","last_active":"2024-04-10T03:23:34.155786+0000","last_peered":"2024-04-10T03:23:34.155786+0000","last_clean":"2024-04-10T03:23:34.155786+0000","last_became_active":"2024-04-10T03:23:34.155415+0000","last_became_peered":"2024-04-10T03:23:34.155415+0000","last_unstale":"2024-04-10T03:23:34.155786+0000","last_undegraded":"2024-04-10T03:23:34.155786+0000","last_fullsized":"2024-04-10T03:23:34.155786+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:53:24.653149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153544+0000","last_change":"2024-04-10T03:23:34.153544+0000","last_active":"2024-04-10T03:23:34.153544+0000","last_peered":"2024-04-10T03:23:34.153544+0000","last_clean":"2024-04-10T03:23:34.153544+0000","last_became_active":"2024-04-10T03:23:34.152793+0000","last_became_peered":"2024-04-10T03:23:34.152793+0000","last_unstale":"2024-04-10T03:23:34.153544+0000","last_undegraded":"2024-04-10T03:23:34.153544+0000","last_fullsized":"2024-04-10T03:23:34.153544+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:00:27.787972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.140500+0000","last_change":"2024-04-10T03:22:58.819243+0000","last_active":"2024-04-10T03:23:34.140500+0000","last_peered":"2024-04-10T03:23:34.140500+0000","last_clean":"2024-04-10T03:23:34.140500+0000","last_became_active":"2024-04-10T03:22:58.818931+0000","last_became_peered":"2024-04-10T03:22:58.818931+0000","last_unstale":"2024-04-10T03:23:34.140500+0000","last_undegraded":"2024-04-10T03:23:34.140500+0000","last_fullsized":"2024-04-10T03:23:34.140500+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:20:44.055340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.135889+0000","last_change":"2024-04-10T03:03:52.176719+0000","last_active":"2024-04-10T03:23:33.135889+0000","last_peered":"2024-04-10T03:23:33.135889+0000","last_clean":"2024-04-10T03:23:33.135889+0000","last_became_active":"2024-04-10T03:03:52.176341+0000","last_became_peered":"2024-04-10T03:03:52.176341+0000","last_unstale":"2024-04-10T03:23:33.135889+0000","last_undegraded":"2024-04-10T03:23:33.135889+0000","last_fullsized":"2024-04-10T03:23:33.135889+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:43.066934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134756+0000","last_change":"2024-04-10T03:19:06.510620+0000","last_active":"2024-04-10T03:23:33.134756+0000","last_peered":"2024-04-10T03:23:33.134756+0000","last_clean":"2024-04-10T03:23:33.134756+0000","last_became_active":"2024-04-10T03:19:06.510274+0000","last_became_peered":"2024-04-10T03:19:06.510274+0000","last_unstale":"2024-04-10T03:23:33.134756+0000","last_undegraded":"2024-04-10T03:23:33.134756+0000","last_fullsized":"2024-04-10T03:23:33.134756+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:05:39.645055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139198+0000","last_change":"2024-04-10T03:22:58.819222+0000","last_active":"2024-04-10T03:23:33.139198+0000","last_peered":"2024-04-10T03:23:33.139198+0000","last_clean":"2024-04-10T03:23:33.139198+0000","last_became_active":"2024-04-10T03:22:58.818712+0000","last_became_peered":"2024-04-10T03:22:58.818712+0000","last_unstale":"2024-04-10T03:23:33.139198+0000","last_undegraded":"2024-04-10T03:23:33.139198+0000","last_fullsized":"2024-04-10T03:23:33.139198+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:09:03.181232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152117+0000","last_change":"2024-04-10T03:23:34.152117+0000","last_active":"2024-04-10T03:23:34.152117+0000","last_peered":"2024-04-10T03:23:34.152117+0000","last_clean":"2024-04-10T03:23:34.152117+0000","last_became_active":"2024-04-10T03:23:34.151730+0000","last_became_peered":"2024-04-10T03:23:34.151730+0000","last_unstale":"2024-04-10T03:23:34.152117+0000","last_undegraded":"2024-04-10T03:23:34.152117+0000","last_fullsized":"2024-04-10T03:23:34.152117+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:58:24.389948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152110+0000","last_change":"2024-04-10T03:23:34.150165+0000","last_active":"2024-04-10T03:23:34.152110+0000","last_peered":"2024-04-10T03:23:34.152110+0000","last_clean":"2024-04-10T03:23:34.152110+0000","last_became_active":"2024-04-10T03:23:34.149800+0000","last_became_peered":"2024-04-10T03:23:34.149800+0000","last_unstale":"2024-04-10T03:23:34.152110+0000","last_undegraded":"2024-04-10T03:23:34.152110+0000","last_fullsized":"2024-04-10T03:23:34.152110+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:39:23.070157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151191+0000","last_change":"2024-04-10T03:23:34.151191+0000","last_active":"2024-04-10T03:23:34.151191+0000","last_peered":"2024-04-10T03:23:34.151191+0000","last_clean":"2024-04-10T03:23:34.151191+0000","last_became_active":"2024-04-10T03:23:34.150769+0000","last_became_peered":"2024-04-10T03:23:34.150769+0000","last_unstale":"2024-04-10T03:23:34.151191+0000","last_undegraded":"2024-04-10T03:23:34.151191+0000","last_fullsized":"2024-04-10T03:23:34.151191+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:32:05.870647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143094+0000","last_change":"2024-04-10T03:03:52.171594+0000","last_active":"2024-04-10T03:23:33.143094+0000","last_peered":"2024-04-10T03:23:33.143094+0000","last_clean":"2024-04-10T03:23:33.143094+0000","last_became_active":"2024-04-10T03:03:52.171314+0000","last_became_peered":"2024-04-10T03:03:52.171314+0000","last_unstale":"2024-04-10T03:23:33.143094+0000","last_undegraded":"2024-04-10T03:23:33.143094+0000","last_fullsized":"2024-04-10T03:23:33.143094+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:37:55.120481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152376+0000","last_change":"2024-04-10T03:23:34.151952+0000","last_active":"2024-04-10T03:23:34.152376+0000","last_peered":"2024-04-10T03:23:34.152376+0000","last_clean":"2024-04-10T03:23:34.152376+0000","last_became_active":"2024-04-10T03:23:34.151635+0000","last_became_peered":"2024-04-10T03:23:34.151635+0000","last_unstale":"2024-04-10T03:23:34.152376+0000","last_undegraded":"2024-04-10T03:23:34.152376+0000","last_fullsized":"2024-04-10T03:23:34.152376+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:32:13.312002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":106,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.140340+0000","last_change":"2024-04-10T03:14:54.251381+0000","last_active":"2024-04-10T03:23:33.140340+0000","last_peered":"2024-04-10T03:23:33.140340+0000","last_clean":"2024-04-10T03:23:33.140340+0000","last_became_active":"2024-04-10T03:14:54.250834+0000","last_became_peered":"2024-04-10T03:14:54.250834+0000","last_unstale":"2024-04-10T03:23:33.140340+0000","last_undegraded":"2024-04-10T03:23:33.140340+0000","last_fullsized":"2024-04-10T03:23:33.140340+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:42:06.054546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151212+0000","last_change":"2024-04-10T03:23:34.151212+0000","last_active":"2024-04-10T03:23:34.151212+0000","last_peered":"2024-04-10T03:23:34.151212+0000","last_clean":"2024-04-10T03:23:34.151212+0000","last_became_active":"2024-04-10T03:23:34.150762+0000","last_became_peered":"2024-04-10T03:23:34.150762+0000","last_unstale":"2024-04-10T03:23:34.151212+0000","last_undegraded":"2024-04-10T03:23:34.151212+0000","last_fullsized":"2024-04-10T03:23:34.151212+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:21:33.466244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.141597+0000","last_change":"2024-04-10T03:14:54.256468+0000","last_active":"2024-04-10T03:23:33.141597+0000","last_peered":"2024-04-10T03:23:33.141597+0000","last_clean":"2024-04-10T03:23:33.141597+0000","last_became_active":"2024-04-10T03:14:54.255966+0000","last_became_peered":"2024-04-10T03:14:54.255966+0000","last_unstale":"2024-04-10T03:23:33.141597+0000","last_undegraded":"2024-04-10T03:23:33.141597+0000","last_fullsized":"2024-04-10T03:23:33.141597+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:33:52.775163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133944+0000","last_change":"2024-04-10T03:03:53.200529+0000","last_active":"2024-04-10T03:23:33.133944+0000","last_peered":"2024-04-10T03:23:33.133944+0000","last_clean":"2024-04-10T03:23:33.133944+0000","last_became_active":"2024-04-10T03:03:53.199080+0000","last_became_peered":"2024-04-10T03:03:53.199080+0000","last_unstale":"2024-04-10T03:23:33.133944+0000","last_undegraded":"2024-04-10T03:23:33.133944+0000","last_fullsized":"2024-04-10T03:23:33.133944+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T15:00:49.737785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143820+0000","last_change":"2024-04-10T03:19:06.503903+0000","last_active":"2024-04-10T03:23:33.143820+0000","last_peered":"2024-04-10T03:23:33.143820+0000","last_clean":"2024-04-10T03:23:33.143820+0000","last_became_active":"2024-04-10T03:19:06.503565+0000","last_became_peered":"2024-04-10T03:19:06.503565+0000","last_unstale":"2024-04-10T03:23:33.143820+0000","last_undegraded":"2024-04-10T03:23:33.143820+0000","last_fullsized":"2024-04-10T03:23:33.143820+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:04:44.209590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.150887+0000","last_change":"2024-04-10T03:23:34.150887+0000","last_active":"2024-04-10T03:23:34.150887+0000","last_peered":"2024-04-10T03:23:34.150887+0000","last_clean":"2024-04-10T03:23:34.150887+0000","last_became_active":"2024-04-10T03:23:34.150540+0000","last_became_peered":"2024-04-10T03:23:34.150540+0000","last_unstale":"2024-04-10T03:23:34.150887+0000","last_undegraded":"2024-04-10T03:23:34.150887+0000","last_fullsized":"2024-04-10T03:23:34.150887+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:24:21.573648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139127+0000","last_change":"2024-04-10T03:03:52.170665+0000","last_active":"2024-04-10T03:23:33.139127+0000","last_peered":"2024-04-10T03:23:33.139127+0000","last_clean":"2024-04-10T03:23:33.139127+0000","last_became_active":"2024-04-10T03:03:52.170385+0000","last_became_peered":"2024-04-10T03:03:52.170385+0000","last_unstale":"2024-04-10T03:23:33.139127+0000","last_undegraded":"2024-04-10T03:23:33.139127+0000","last_fullsized":"2024-04-10T03:23:33.139127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:36:28.794484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.142580+0000","last_change":"2024-04-10T03:19:06.498472+0000","last_active":"2024-04-10T03:23:33.142580+0000","last_peered":"2024-04-10T03:23:33.142580+0000","last_clean":"2024-04-10T03:23:33.142580+0000","last_became_active":"2024-04-10T03:19:06.498130+0000","last_became_peered":"2024-04-10T03:19:06.498130+0000","last_unstale":"2024-04-10T03:23:33.142580+0000","last_undegraded":"2024-04-10T03:23:33.142580+0000","last_fullsized":"2024-04-10T03:23:33.142580+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:31:20.254590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153444+0000","last_change":"2024-04-10T03:23:34.153444+0000","last_active":"2024-04-10T03:23:34.153444+0000","last_peered":"2024-04-10T03:23:34.153444+0000","last_clean":"2024-04-10T03:23:34.153444+0000","last_became_active":"2024-04-10T03:23:34.153109+0000","last_became_peered":"2024-04-10T03:23:34.153109+0000","last_unstale":"2024-04-10T03:23:34.153444+0000","last_undegraded":"2024-04-10T03:23:34.153444+0000","last_fullsized":"2024-04-10T03:23:34.153444+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:23:44.895576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139198+0000","last_change":"2024-04-10T03:19:06.512811+0000","last_active":"2024-04-10T03:23:34.139198+0000","last_peered":"2024-04-10T03:23:34.139198+0000","last_clean":"2024-04-10T03:23:34.139198+0000","last_became_active":"2024-04-10T03:19:06.512465+0000","last_became_peered":"2024-04-10T03:19:06.512465+0000","last_unstale":"2024-04-10T03:23:34.139198+0000","last_undegraded":"2024-04-10T03:23:34.139198+0000","last_fullsized":"2024-04-10T03:23:34.139198+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:57:23.548014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":109,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.158338+0000","last_change":"2024-04-10T03:23:34.158338+0000","last_active":"2024-04-10T03:23:34.158338+0000","last_peered":"2024-04-10T03:23:34.158338+0000","last_clean":"2024-04-10T03:23:34.158338+0000","last_became_active":"2024-04-10T03:23:34.157304+0000","last_became_peered":"2024-04-10T03:23:34.157304+0000","last_unstale":"2024-04-10T03:23:34.158338+0000","last_undegraded":"2024-04-10T03:23:34.158338+0000","last_fullsized":"2024-04-10T03:23:34.158338+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:22:20.352365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152209+0000","last_change":"2024-04-10T03:23:34.152209+0000","last_active":"2024-04-10T03:23:34.152209+0000","last_peered":"2024-04-10T03:23:34.152209+0000","last_clean":"2024-04-10T03:23:34.152209+0000","last_became_active":"2024-04-10T03:23:34.151856+0000","last_became_peered":"2024-04-10T03:23:34.151856+0000","last_unstale":"2024-04-10T03:23:34.152209+0000","last_undegraded":"2024-04-10T03:23:34.152209+0000","last_fullsized":"2024-04-10T03:23:34.152209+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:02:26.611265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.142812+0000","last_change":"2024-04-10T03:19:06.500320+0000","last_active":"2024-04-10T03:23:33.142812+0000","last_peered":"2024-04-10T03:23:33.142812+0000","last_clean":"2024-04-10T03:23:33.142812+0000","last_became_active":"2024-04-10T03:19:06.500001+0000","last_became_peered":"2024-04-10T03:19:06.500001+0000","last_unstale":"2024-04-10T03:23:33.142812+0000","last_undegraded":"2024-04-10T03:23:33.142812+0000","last_fullsized":"2024-04-10T03:23:33.142812+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:44:33.676068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.136333+0000","last_change":"2024-04-10T03:22:58.818406+0000","last_active":"2024-04-10T03:23:33.136333+0000","last_peered":"2024-04-10T03:23:33.136333+0000","last_clean":"2024-04-10T03:23:33.136333+0000","last_became_active":"2024-04-10T03:22:58.818060+0000","last_became_peered":"2024-04-10T03:22:58.818060+0000","last_unstale":"2024-04-10T03:23:33.136333+0000","last_undegraded":"2024-04-10T03:23:33.136333+0000","last_fullsized":"2024-04-10T03:23:33.136333+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:03:06.653280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.135105+0000","last_change":"2024-04-10T03:22:58.822444+0000","last_active":"2024-04-10T03:23:33.135105+0000","last_peered":"2024-04-10T03:23:33.135105+0000","last_clean":"2024-04-10T03:23:33.135105+0000","last_became_active":"2024-04-10T03:22:58.822064+0000","last_became_peered":"2024-04-10T03:22:58.822064+0000","last_unstale":"2024-04-10T03:23:33.135105+0000","last_undegraded":"2024-04-10T03:23:33.135105+0000","last_fullsized":"2024-04-10T03:23:33.135105+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:31:29.552226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.136960+0000","last_change":"2024-04-10T03:19:06.511142+0000","last_active":"2024-04-10T03:23:33.136960+0000","last_peered":"2024-04-10T03:23:33.136960+0000","last_clean":"2024-04-10T03:23:33.136960+0000","last_became_active":"2024-04-10T03:19:06.510840+0000","last_became_peered":"2024-04-10T03:19:06.510840+0000","last_unstale":"2024-04-10T03:23:33.136960+0000","last_undegraded":"2024-04-10T03:23:33.136960+0000","last_fullsized":"2024-04-10T03:23:33.136960+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:13:40.385441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151002+0000","last_change":"2024-04-10T03:23:34.151002+0000","last_active":"2024-04-10T03:23:34.151002+0000","last_peered":"2024-04-10T03:23:34.151002+0000","last_clean":"2024-04-10T03:23:34.151002+0000","last_became_active":"2024-04-10T03:23:34.150668+0000","last_became_peered":"2024-04-10T03:23:34.150668+0000","last_unstale":"2024-04-10T03:23:34.151002+0000","last_undegraded":"2024-04-10T03:23:34.151002+0000","last_fullsized":"2024-04-10T03:23:34.151002+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:24:55.948167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.149578+0000","last_change":"2024-04-10T03:23:34.148938+0000","last_active":"2024-04-10T03:23:34.149578+0000","last_peered":"2024-04-10T03:23:34.149578+0000","last_clean":"2024-04-10T03:23:34.149578+0000","last_became_active":"2024-04-10T03:23:34.148594+0000","last_became_peered":"2024-04-10T03:23:34.148594+0000","last_unstale":"2024-04-10T03:23:34.149578+0000","last_undegraded":"2024-04-10T03:23:34.149578+0000","last_fullsized":"2024-04-10T03:23:34.149578+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:54:24.394886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141335+0000","last_change":"2024-04-10T03:19:06.502575+0000","last_active":"2024-04-10T03:23:34.141335+0000","last_peered":"2024-04-10T03:23:34.141335+0000","last_clean":"2024-04-10T03:23:34.141335+0000","last_became_active":"2024-04-10T03:19:06.502292+0000","last_became_peered":"2024-04-10T03:19:06.502292+0000","last_unstale":"2024-04-10T03:23:34.141335+0000","last_undegraded":"2024-04-10T03:23:34.141335+0000","last_fullsized":"2024-04-10T03:23:34.141335+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:33:26.735169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151449+0000","last_change":"2024-04-10T03:23:34.151449+0000","last_active":"2024-04-10T03:23:34.151449+0000","last_peered":"2024-04-10T03:23:34.151449+0000","last_clean":"2024-04-10T03:23:34.151449+0000","last_became_active":"2024-04-10T03:23:34.151115+0000","last_became_peered":"2024-04-10T03:23:34.151115+0000","last_unstale":"2024-04-10T03:23:34.151449+0000","last_undegraded":"2024-04-10T03:23:34.151449+0000","last_fullsized":"2024-04-10T03:23:34.151449+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:34:25.299234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.140247+0000","last_change":"2024-04-10T03:19:06.508818+0000","last_active":"2024-04-10T03:23:34.140247+0000","last_peered":"2024-04-10T03:23:34.140247+0000","last_clean":"2024-04-10T03:23:34.140247+0000","last_became_active":"2024-04-10T03:19:06.507140+0000","last_became_peered":"2024-04-10T03:19:06.507140+0000","last_unstale":"2024-04-10T03:23:34.140247+0000","last_undegraded":"2024-04-10T03:23:34.140247+0000","last_fullsized":"2024-04-10T03:23:34.140247+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:00:54.482723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133932+0000","last_change":"2024-04-10T03:19:06.501152+0000","last_active":"2024-04-10T03:23:33.133932+0000","last_peered":"2024-04-10T03:23:33.133932+0000","last_clean":"2024-04-10T03:23:33.133932+0000","last_became_active":"2024-04-10T03:19:06.500732+0000","last_became_peered":"2024-04-10T03:19:06.500732+0000","last_unstale":"2024-04-10T03:23:33.133932+0000","last_undegraded":"2024-04-10T03:23:33.133932+0000","last_fullsized":"2024-04-10T03:23:33.133932+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:20:56.343876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134849+0000","last_change":"2024-04-10T03:14:54.260506+0000","last_active":"2024-04-10T03:23:33.134849+0000","last_peered":"2024-04-10T03:23:33.134849+0000","last_clean":"2024-04-10T03:23:33.134849+0000","last_became_active":"2024-04-10T03:14:54.260180+0000","last_became_peered":"2024-04-10T03:14:54.260180+0000","last_unstale":"2024-04-10T03:23:33.134849+0000","last_undegraded":"2024-04-10T03:23:33.134849+0000","last_fullsized":"2024-04-10T03:23:33.134849+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:38:08.636706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153406+0000","last_change":"2024-04-10T03:23:34.153406+0000","last_active":"2024-04-10T03:23:34.153406+0000","last_peered":"2024-04-10T03:23:34.153406+0000","last_clean":"2024-04-10T03:23:34.153406+0000","last_became_active":"2024-04-10T03:23:34.152999+0000","last_became_peered":"2024-04-10T03:23:34.152999+0000","last_unstale":"2024-04-10T03:23:34.153406+0000","last_undegraded":"2024-04-10T03:23:34.153406+0000","last_fullsized":"2024-04-10T03:23:34.153406+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:02:54.989305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":89,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151258+0000","last_change":"2024-04-10T03:23:34.151258+0000","last_active":"2024-04-10T03:23:34.151258+0000","last_peered":"2024-04-10T03:23:34.151258+0000","last_clean":"2024-04-10T03:23:34.151258+0000","last_became_active":"2024-04-10T03:23:34.150937+0000","last_became_peered":"2024-04-10T03:23:34.150937+0000","last_unstale":"2024-04-10T03:23:34.151258+0000","last_undegraded":"2024-04-10T03:23:34.151258+0000","last_fullsized":"2024-04-10T03:23:34.151258+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:23:19.028348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139415+0000","last_change":"2024-04-10T03:22:58.820025+0000","last_active":"2024-04-10T03:23:34.139415+0000","last_peered":"2024-04-10T03:23:34.139415+0000","last_clean":"2024-04-10T03:23:34.139415+0000","last_became_active":"2024-04-10T03:22:58.819681+0000","last_became_peered":"2024-04-10T03:22:58.819681+0000","last_unstale":"2024-04-10T03:23:34.139415+0000","last_undegraded":"2024-04-10T03:23:34.139415+0000","last_fullsized":"2024-04-10T03:23:34.139415+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:02:29.026294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141439+0000","last_change":"2024-04-10T03:19:06.518701+0000","last_active":"2024-04-10T03:23:34.141439+0000","last_peered":"2024-04-10T03:23:34.141439+0000","last_clean":"2024-04-10T03:23:34.141439+0000","last_became_active":"2024-04-10T03:19:06.508984+0000","last_became_peered":"2024-04-10T03:19:06.508984+0000","last_unstale":"2024-04-10T03:23:34.141439+0000","last_undegraded":"2024-04-10T03:23:34.141439+0000","last_fullsized":"2024-04-10T03:23:34.141439+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:21:05.066857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134038+0000","last_change":"2024-04-10T03:22:58.818093+0000","last_active":"2024-04-10T03:23:33.134038+0000","last_peered":"2024-04-10T03:23:33.134038+0000","last_clean":"2024-04-10T03:23:33.134038+0000","last_became_active":"2024-04-10T03:22:58.817757+0000","last_became_peered":"2024-04-10T03:22:58.817757+0000","last_unstale":"2024-04-10T03:23:33.134038+0000","last_undegraded":"2024-04-10T03:23:33.134038+0000","last_fullsized":"2024-04-10T03:23:33.134038+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:26:03.034823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":478,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.138521+0000","last_change":"2024-04-10T03:14:54.256102+0000","last_active":"2024-04-10T03:23:33.138521+0000","last_peered":"2024-04-10T03:23:33.138521+0000","last_clean":"2024-04-10T03:23:33.138521+0000","last_became_active":"2024-04-10T03:14:54.255639+0000","last_became_peered":"2024-04-10T03:14:54.255639+0000","last_unstale":"2024-04-10T03:23:33.138521+0000","last_undegraded":"2024-04-10T03:23:33.138521+0000","last_fullsized":"2024-04-10T03:23:33.138521+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:55:00.311119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155980+0000","last_change":"2024-04-10T03:23:34.155980+0000","last_active":"2024-04-10T03:23:34.155980+0000","last_peered":"2024-04-10T03:23:34.155980+0000","last_clean":"2024-04-10T03:23:34.155980+0000","last_became_active":"2024-04-10T03:23:34.155532+0000","last_became_peered":"2024-04-10T03:23:34.155532+0000","last_unstale":"2024-04-10T03:23:34.155980+0000","last_undegraded":"2024-04-10T03:23:34.155980+0000","last_fullsized":"2024-04-10T03:23:34.155980+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:49:56.439170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139967+0000","last_change":"2024-04-10T03:03:53.201722+0000","last_active":"2024-04-10T03:23:33.139967+0000","last_peered":"2024-04-10T03:23:33.139967+0000","last_clean":"2024-04-10T03:23:33.139967+0000","last_became_active":"2024-04-10T03:03:53.201394+0000","last_became_peered":"2024-04-10T03:03:53.201394+0000","last_unstale":"2024-04-10T03:23:33.139967+0000","last_undegraded":"2024-04-10T03:23:33.139967+0000","last_fullsized":"2024-04-10T03:23:33.139967+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:16.107170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.141702+0000","last_change":"2024-04-10T03:19:06.502901+0000","last_active":"2024-04-10T03:23:33.141702+0000","last_peered":"2024-04-10T03:23:33.141702+0000","last_clean":"2024-04-10T03:23:33.141702+0000","last_became_active":"2024-04-10T03:19:06.502562+0000","last_became_peered":"2024-04-10T03:19:06.502562+0000","last_unstale":"2024-04-10T03:23:33.141702+0000","last_undegraded":"2024-04-10T03:23:33.141702+0000","last_fullsized":"2024-04-10T03:23:33.141702+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:42:35.746508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.154331+0000","last_change":"2024-04-10T03:23:34.154331+0000","last_active":"2024-04-10T03:23:34.154331+0000","last_peered":"2024-04-10T03:23:34.154331+0000","last_clean":"2024-04-10T03:23:34.154331+0000","last_became_active":"2024-04-10T03:23:34.153986+0000","last_became_peered":"2024-04-10T03:23:34.153986+0000","last_unstale":"2024-04-10T03:23:34.154331+0000","last_undegraded":"2024-04-10T03:23:34.154331+0000","last_fullsized":"2024-04-10T03:23:34.154331+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:17:50.203014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":141,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139445+0000","last_change":"2024-04-10T03:22:58.818486+0000","last_active":"2024-04-10T03:23:34.139445+0000","last_peered":"2024-04-10T03:23:34.139445+0000","last_clean":"2024-04-10T03:23:34.139445+0000","last_became_active":"2024-04-10T03:22:58.818170+0000","last_became_peered":"2024-04-10T03:22:58.818170+0000","last_unstale":"2024-04-10T03:23:34.139445+0000","last_undegraded":"2024-04-10T03:23:34.139445+0000","last_fullsized":"2024-04-10T03:23:34.139445+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:03:41.192593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139271+0000","last_change":"2024-04-10T03:14:54.287221+0000","last_active":"2024-04-10T03:23:33.139271+0000","last_peered":"2024-04-10T03:23:33.139271+0000","last_clean":"2024-04-10T03:23:33.139271+0000","last_became_active":"2024-04-10T03:14:54.286933+0000","last_became_peered":"2024-04-10T03:14:54.286933+0000","last_unstale":"2024-04-10T03:23:33.139271+0000","last_undegraded":"2024-04-10T03:23:33.139271+0000","last_fullsized":"2024-04-10T03:23:33.139271+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:59:12.810453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141757+0000","last_change":"2024-04-10T03:22:58.822527+0000","last_active":"2024-04-10T03:23:34.141757+0000","last_peered":"2024-04-10T03:23:34.141757+0000","last_clean":"2024-04-10T03:23:34.141757+0000","last_became_active":"2024-04-10T03:22:58.822195+0000","last_became_peered":"2024-04-10T03:22:58.822195+0000","last_unstale":"2024-04-10T03:23:34.141757+0000","last_undegraded":"2024-04-10T03:23:34.141757+0000","last_fullsized":"2024-04-10T03:23:34.141757+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:55:20.736706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.138878+0000","last_change":"2024-04-10T03:14:54.257574+0000","last_active":"2024-04-10T03:23:34.138878+0000","last_peered":"2024-04-10T03:23:34.138878+0000","last_clean":"2024-04-10T03:23:34.138878+0000","last_became_active":"2024-04-10T03:14:54.256568+0000","last_became_peered":"2024-04-10T03:14:54.256568+0000","last_unstale":"2024-04-10T03:23:34.138878+0000","last_undegraded":"2024-04-10T03:23:34.138878+0000","last_fullsized":"2024-04-10T03:23:34.138878+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:51:23.401981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":109,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156225+0000","last_change":"2024-04-10T03:23:34.156225+0000","last_active":"2024-04-10T03:23:34.156225+0000","last_peered":"2024-04-10T03:23:34.156225+0000","last_clean":"2024-04-10T03:23:34.156225+0000","last_became_active":"2024-04-10T03:23:34.155775+0000","last_became_peered":"2024-04-10T03:23:34.155775+0000","last_unstale":"2024-04-10T03:23:34.156225+0000","last_undegraded":"2024-04-10T03:23:34.156225+0000","last_fullsized":"2024-04-10T03:23:34.156225+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:53.125600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139985+0000","last_change":"2024-04-10T03:19:06.514780+0000","last_active":"2024-04-10T03:23:34.139985+0000","last_peered":"2024-04-10T03:23:34.139985+0000","last_clean":"2024-04-10T03:23:34.139985+0000","last_became_active":"2024-04-10T03:19:06.514387+0000","last_became_peered":"2024-04-10T03:19:06.514387+0000","last_unstale":"2024-04-10T03:23:34.139985+0000","last_undegraded":"2024-04-10T03:23:34.139985+0000","last_fullsized":"2024-04-10T03:23:34.139985+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:00:08.784016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156596+0000","last_change":"2024-04-10T03:23:34.156596+0000","last_active":"2024-04-10T03:23:34.156596+0000","last_peered":"2024-04-10T03:23:34.156596+0000","last_clean":"2024-04-10T03:23:34.156596+0000","last_became_active":"2024-04-10T03:23:34.156276+0000","last_became_peered":"2024-04-10T03:23:34.156276+0000","last_unstale":"2024-04-10T03:23:34.156596+0000","last_undegraded":"2024-04-10T03:23:34.156596+0000","last_fullsized":"2024-04-10T03:23:34.156596+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:54:35.146259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143007+0000","last_change":"2024-04-10T03:22:58.819134+0000","last_active":"2024-04-10T03:23:33.143007+0000","last_peered":"2024-04-10T03:23:33.143007+0000","last_clean":"2024-04-10T03:23:33.143007+0000","last_became_active":"2024-04-10T03:22:58.818810+0000","last_became_peered":"2024-04-10T03:22:58.818810+0000","last_unstale":"2024-04-10T03:23:33.143007+0000","last_undegraded":"2024-04-10T03:23:33.143007+0000","last_fullsized":"2024-04-10T03:23:33.143007+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:46:50.340812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139478+0000","last_change":"2024-04-10T03:19:06.512958+0000","last_active":"2024-04-10T03:23:34.139478+0000","last_peered":"2024-04-10T03:23:34.139478+0000","last_clean":"2024-04-10T03:23:34.139478+0000","last_became_active":"2024-04-10T03:19:06.512542+0000","last_became_peered":"2024-04-10T03:19:06.512542+0000","last_unstale":"2024-04-10T03:23:34.139478+0000","last_undegraded":"2024-04-10T03:23:34.139478+0000","last_fullsized":"2024-04-10T03:23:34.139478+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:09.385448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133994+0000","last_change":"2024-04-10T03:03:53.193384+0000","last_active":"2024-04-10T03:23:33.133994+0000","last_peered":"2024-04-10T03:23:33.133994+0000","last_clean":"2024-04-10T03:23:33.133994+0000","last_became_active":"2024-04-10T03:03:53.193048+0000","last_became_peered":"2024-04-10T03:03:53.193048+0000","last_unstale":"2024-04-10T03:23:33.133994+0000","last_undegraded":"2024-04-10T03:23:33.133994+0000","last_fullsized":"2024-04-10T03:23:33.133994+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:10:41.808720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139269+0000","last_change":"2024-04-10T03:14:54.259178+0000","last_active":"2024-04-10T03:23:34.139269+0000","last_peered":"2024-04-10T03:23:34.139269+0000","last_clean":"2024-04-10T03:23:34.139269+0000","last_became_active":"2024-04-10T03:14:54.257620+0000","last_became_peered":"2024-04-10T03:14:54.257620+0000","last_unstale":"2024-04-10T03:23:34.139269+0000","last_undegraded":"2024-04-10T03:23:34.139269+0000","last_fullsized":"2024-04-10T03:23:34.139269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:23:43.049692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156329+0000","last_change":"2024-04-10T03:23:34.156329+0000","last_active":"2024-04-10T03:23:34.156329+0000","last_peered":"2024-04-10T03:23:34.156329+0000","last_clean":"2024-04-10T03:23:34.156329+0000","last_became_active":"2024-04-10T03:23:34.155926+0000","last_became_peered":"2024-04-10T03:23:34.155926+0000","last_unstale":"2024-04-10T03:23:34.156329+0000","last_undegraded":"2024-04-10T03:23:34.156329+0000","last_fullsized":"2024-04-10T03:23:34.156329+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:29:38.452599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134208+0000","last_change":"2024-04-10T03:19:06.501427+0000","last_active":"2024-04-10T03:23:33.134208+0000","last_peered":"2024-04-10T03:23:33.134208+0000","last_clean":"2024-04-10T03:23:33.134208+0000","last_became_active":"2024-04-10T03:19:06.500055+0000","last_became_peered":"2024-04-10T03:19:06.500055+0000","last_unstale":"2024-04-10T03:23:33.134208+0000","last_undegraded":"2024-04-10T03:23:33.134208+0000","last_fullsized":"2024-04-10T03:23:33.134208+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:11:52.273430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153366+0000","last_change":"2024-04-10T03:23:34.153366+0000","last_active":"2024-04-10T03:23:34.153366+0000","last_peered":"2024-04-10T03:23:34.153366+0000","last_clean":"2024-04-10T03:23:34.153366+0000","last_became_active":"2024-04-10T03:23:34.152839+0000","last_became_peered":"2024-04-10T03:23:34.152839+0000","last_unstale":"2024-04-10T03:23:34.153366+0000","last_undegraded":"2024-04-10T03:23:34.153366+0000","last_fullsized":"2024-04-10T03:23:34.153366+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:17:50.987961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134334+0000","last_change":"2024-04-10T03:03:52.169154+0000","last_active":"2024-04-10T03:23:33.134334+0000","last_peered":"2024-04-10T03:23:33.134334+0000","last_clean":"2024-04-10T03:23:33.134334+0000","last_became_active":"2024-04-10T03:03:52.168854+0000","last_became_peered":"2024-04-10T03:03:52.168854+0000","last_unstale":"2024-04-10T03:23:33.134334+0000","last_undegraded":"2024-04-10T03:23:33.134334+0000","last_fullsized":"2024-04-10T03:23:33.134334+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:01:21.091245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.148758+0000","last_change":"2024-04-10T03:23:34.148758+0000","last_active":"2024-04-10T03:23:34.148758+0000","last_peered":"2024-04-10T03:23:34.148758+0000","last_clean":"2024-04-10T03:23:34.148758+0000","last_became_active":"2024-04-10T03:23:34.148428+0000","last_became_peered":"2024-04-10T03:23:34.148428+0000","last_unstale":"2024-04-10T03:23:34.148758+0000","last_undegraded":"2024-04-10T03:23:34.148758+0000","last_fullsized":"2024-04-10T03:23:34.148758+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:41:21.082952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155949+0000","last_change":"2024-04-10T03:23:34.155949+0000","last_active":"2024-04-10T03:23:34.155949+0000","last_peered":"2024-04-10T03:23:34.155949+0000","last_clean":"2024-04-10T03:23:34.155949+0000","last_became_active":"2024-04-10T03:23:34.155340+0000","last_became_peered":"2024-04-10T03:23:34.155340+0000","last_unstale":"2024-04-10T03:23:34.155949+0000","last_undegraded":"2024-04-10T03:23:34.155949+0000","last_fullsized":"2024-04-10T03:23:34.155949+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:59:23.588701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156934+0000","last_change":"2024-04-10T03:23:34.156934+0000","last_active":"2024-04-10T03:23:34.156934+0000","last_peered":"2024-04-10T03:23:34.156934+0000","last_clean":"2024-04-10T03:23:34.156934+0000","last_became_active":"2024-04-10T03:23:34.156512+0000","last_became_peered":"2024-04-10T03:23:34.156512+0000","last_unstale":"2024-04-10T03:23:34.156934+0000","last_undegraded":"2024-04-10T03:23:34.156934+0000","last_fullsized":"2024-04-10T03:23:34.156934+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:04:08.142967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139099+0000","last_change":"2024-04-10T03:22:58.819167+0000","last_active":"2024-04-10T03:23:34.139099+0000","last_peered":"2024-04-10T03:23:34.139099+0000","last_clean":"2024-04-10T03:23:34.139099+0000","last_became_active":"2024-04-10T03:22:58.818808+0000","last_became_peered":"2024-04-10T03:22:58.818808+0000","last_unstale":"2024-04-10T03:23:34.139099+0000","last_undegraded":"2024-04-10T03:23:34.139099+0000","last_fullsized":"2024-04-10T03:23:34.139099+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:45:04.307428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133994+0000","last_change":"2024-04-10T03:19:06.511070+0000","last_active":"2024-04-10T03:23:33.133994+0000","last_peered":"2024-04-10T03:23:33.133994+0000","last_clean":"2024-04-10T03:23:33.133994+0000","last_became_active":"2024-04-10T03:19:06.510581+0000","last_became_peered":"2024-04-10T03:19:06.510581+0000","last_unstale":"2024-04-10T03:23:33.133994+0000","last_undegraded":"2024-04-10T03:23:33.133994+0000","last_fullsized":"2024-04-10T03:23:33.133994+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:10:59.440513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13613,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234256,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592028,"kb_used_data":2192,"kb_used_omap":1,"kb_used_meta":589822,"kb_avail":93153124,"statfs":{"total":95995035648,"available":95388798976,"internally_reserved":0,"allocated":2244608,"data_stored":1485199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":603978072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495894,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307728,"kb_used_data":2176,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93437424,"statfs":{"total":95995035648,"available":95679922176,"internally_reserved":0,"allocated":2228224,"data_stored":1484772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":114,"seq":489626271811,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":184712,"kb_used_data":1588,"kb_used_omap":5,"kb_used_meta":183098,"kb_avail":93560440,"statfs":{"total":95995035648,"available":95805890560,"internally_reserved":0,"allocated":1626112,"data_stored":890198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":187492631},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024054,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":282000,"kb_used_data":2176,"kb_used_omap":5,"kb_used_meta":279802,"kb_avail":93463152,"statfs":{"total":95995035648,"available":95706267648,"internally_reserved":0,"allocated":2228224,"data_stored":1481182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6053,"internal_metadata":286517339},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":132,"seq":566935683093,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":60304,"kb_used_data":1596,"kb_used_omap":6,"kb_used_meta":58681,"kb_avail":93684848,"statfs":{"total":95995035648,"available":95933284352,"internally_reserved":0,"allocated":1634304,"data_stored":890810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6680,"internal_metadata":60089832},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":137,"seq":588410519566,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78404,"kb_used_data":1592,"kb_used_omap":4,"kb_used_meta":76795,"kb_avail":93666748,"statfs":{"total":95995035648,"available":95914749952,"internally_reserved":0,"allocated":1630208,"data_stored":890288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4642,"internal_metadata":78638558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019167,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603732,"kb_used_data":1608,"kb_used_omap":1,"kb_used_meta":602110,"kb_avail":93141420,"statfs":{"total":95995035648,"available":95376814080,"internally_reserved":0,"allocated":1646592,"data_stored":894811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616560984},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757529,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896712,"kb_used_data":1596,"kb_used_omap":1,"kb_used_meta":895102,"kb_avail":92848440,"statfs":{"total":95995035648,"available":95076802560,"internally_reserved":0,"allocated":1634304,"data_stored":890818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":916584792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4349,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2165,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2938,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:24:35.285 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-10T03:24:35.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:35 smithi152 ceph-mon[88640]: from='client.45227 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:35.541 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:36.291 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:36 smithi102 ceph-mon[98773]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:36.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:36 smithi102 ceph-mon[94535]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:36.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:36 smithi152 ceph-mon[88640]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:37.412 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:37.413 INFO:teuthology.orchestra.run.smithi102.stderr:dumped all 2024-04-10T03:24:38.265 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:38 smithi102 ceph-mon[94535]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:38.265 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:38 smithi102 ceph-mon[94535]: from='client.35433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:38.266 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:38 smithi102 ceph-mon[98773]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:38.266 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:38 smithi102 ceph-mon[98773]: from='client.35433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:38.294 INFO:teuthology.orchestra.run.smithi102.stdout:{"pg_ready":true,"pg_map":{"version":534,"stamp":"2024-04-10T03:24:37.100297+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3005620,"kb_used_data":14524,"kb_used_omap":29,"kb_used_meta":2990946,"kb_avail":746955596,"statfs":{"total":767960285184,"available":764882530304,"internally_reserved":0,"allocated":14872576,"data_stored":8908078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30056,"internal_metadata":3062729368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002799"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134440+0000","last_change":"2024-04-10T03:22:58.820683+0000","last_active":"2024-04-10T03:23:33.134440+0000","last_peered":"2024-04-10T03:23:33.134440+0000","last_clean":"2024-04-10T03:23:33.134440+0000","last_became_active":"2024-04-10T03:22:58.820333+0000","last_became_peered":"2024-04-10T03:22:58.820333+0000","last_unstale":"2024-04-10T03:23:33.134440+0000","last_undegraded":"2024-04-10T03:23:33.134440+0000","last_fullsized":"2024-04-10T03:23:33.134440+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:18:40.834614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.149503+0000","last_change":"2024-04-10T03:23:34.149503+0000","last_active":"2024-04-10T03:23:34.149503+0000","last_peered":"2024-04-10T03:23:34.149503+0000","last_clean":"2024-04-10T03:23:34.149503+0000","last_became_active":"2024-04-10T03:23:34.148752+0000","last_became_peered":"2024-04-10T03:23:34.148752+0000","last_unstale":"2024-04-10T03:23:34.149503+0000","last_undegraded":"2024-04-10T03:23:34.149503+0000","last_fullsized":"2024-04-10T03:23:34.149503+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:11:49.117446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151460+0000","last_change":"2024-04-10T03:23:34.151048+0000","last_active":"2024-04-10T03:23:34.151460+0000","last_peered":"2024-04-10T03:23:34.151460+0000","last_clean":"2024-04-10T03:23:34.151460+0000","last_became_active":"2024-04-10T03:23:34.150595+0000","last_became_peered":"2024-04-10T03:23:34.150595+0000","last_unstale":"2024-04-10T03:23:34.151460+0000","last_undegraded":"2024-04-10T03:23:34.151460+0000","last_fullsized":"2024-04-10T03:23:34.151460+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:24:48.196187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151087+0000","last_change":"2024-04-10T03:23:34.151087+0000","last_active":"2024-04-10T03:23:34.151087+0000","last_peered":"2024-04-10T03:23:34.151087+0000","last_clean":"2024-04-10T03:23:34.151087+0000","last_became_active":"2024-04-10T03:23:34.150662+0000","last_became_peered":"2024-04-10T03:23:34.150662+0000","last_unstale":"2024-04-10T03:23:34.151087+0000","last_undegraded":"2024-04-10T03:23:34.151087+0000","last_fullsized":"2024-04-10T03:23:34.151087+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:43:05.878112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156674+0000","last_change":"2024-04-10T03:23:34.156674+0000","last_active":"2024-04-10T03:23:34.156674+0000","last_peered":"2024-04-10T03:23:34.156674+0000","last_clean":"2024-04-10T03:23:34.156674+0000","last_became_active":"2024-04-10T03:23:34.155850+0000","last_became_peered":"2024-04-10T03:23:34.155850+0000","last_unstale":"2024-04-10T03:23:34.156674+0000","last_undegraded":"2024-04-10T03:23:34.156674+0000","last_fullsized":"2024-04-10T03:23:34.156674+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:20:28.375709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143505+0000","last_change":"2024-04-10T03:19:06.500857+0000","last_active":"2024-04-10T03:23:33.143505+0000","last_peered":"2024-04-10T03:23:33.143505+0000","last_clean":"2024-04-10T03:23:33.143505+0000","last_became_active":"2024-04-10T03:19:06.500122+0000","last_became_peered":"2024-04-10T03:19:06.500122+0000","last_unstale":"2024-04-10T03:23:33.143505+0000","last_undegraded":"2024-04-10T03:23:33.143505+0000","last_fullsized":"2024-04-10T03:23:33.143505+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:11:18.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153369+0000","last_change":"2024-04-10T03:23:34.153056+0000","last_active":"2024-04-10T03:23:34.153369+0000","last_peered":"2024-04-10T03:23:34.153369+0000","last_clean":"2024-04-10T03:23:34.153369+0000","last_became_active":"2024-04-10T03:23:34.152704+0000","last_became_peered":"2024-04-10T03:23:34.152704+0000","last_unstale":"2024-04-10T03:23:34.153369+0000","last_undegraded":"2024-04-10T03:23:34.153369+0000","last_fullsized":"2024-04-10T03:23:34.153369+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:53:15.431644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156190+0000","last_change":"2024-04-10T03:23:34.156190+0000","last_active":"2024-04-10T03:23:34.156190+0000","last_peered":"2024-04-10T03:23:34.156190+0000","last_clean":"2024-04-10T03:23:34.156190+0000","last_became_active":"2024-04-10T03:23:34.155393+0000","last_became_peered":"2024-04-10T03:23:34.155393+0000","last_unstale":"2024-04-10T03:23:34.156190+0000","last_undegraded":"2024-04-10T03:23:34.156190+0000","last_fullsized":"2024-04-10T03:23:34.156190+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:29:53.054637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134831+0000","last_change":"2024-04-10T03:19:06.509704+0000","last_active":"2024-04-10T03:23:33.134831+0000","last_peered":"2024-04-10T03:23:33.134831+0000","last_clean":"2024-04-10T03:23:33.134831+0000","last_became_active":"2024-04-10T03:19:06.509393+0000","last_became_peered":"2024-04-10T03:19:06.509393+0000","last_unstale":"2024-04-10T03:23:33.134831+0000","last_undegraded":"2024-04-10T03:23:33.134831+0000","last_fullsized":"2024-04-10T03:23:33.134831+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:20:15.251509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155786+0000","last_change":"2024-04-10T03:23:34.155786+0000","last_active":"2024-04-10T03:23:34.155786+0000","last_peered":"2024-04-10T03:23:34.155786+0000","last_clean":"2024-04-10T03:23:34.155786+0000","last_became_active":"2024-04-10T03:23:34.155415+0000","last_became_peered":"2024-04-10T03:23:34.155415+0000","last_unstale":"2024-04-10T03:23:34.155786+0000","last_undegraded":"2024-04-10T03:23:34.155786+0000","last_fullsized":"2024-04-10T03:23:34.155786+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:53:24.653149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153544+0000","last_change":"2024-04-10T03:23:34.153544+0000","last_active":"2024-04-10T03:23:34.153544+0000","last_peered":"2024-04-10T03:23:34.153544+0000","last_clean":"2024-04-10T03:23:34.153544+0000","last_became_active":"2024-04-10T03:23:34.152793+0000","last_became_peered":"2024-04-10T03:23:34.152793+0000","last_unstale":"2024-04-10T03:23:34.153544+0000","last_undegraded":"2024-04-10T03:23:34.153544+0000","last_fullsized":"2024-04-10T03:23:34.153544+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:00:27.787972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.140500+0000","last_change":"2024-04-10T03:22:58.819243+0000","last_active":"2024-04-10T03:23:34.140500+0000","last_peered":"2024-04-10T03:23:34.140500+0000","last_clean":"2024-04-10T03:23:34.140500+0000","last_became_active":"2024-04-10T03:22:58.818931+0000","last_became_peered":"2024-04-10T03:22:58.818931+0000","last_unstale":"2024-04-10T03:23:34.140500+0000","last_undegraded":"2024-04-10T03:23:34.140500+0000","last_fullsized":"2024-04-10T03:23:34.140500+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:20:44.055340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":104,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.135889+0000","last_change":"2024-04-10T03:03:52.176719+0000","last_active":"2024-04-10T03:23:33.135889+0000","last_peered":"2024-04-10T03:23:33.135889+0000","last_clean":"2024-04-10T03:23:33.135889+0000","last_became_active":"2024-04-10T03:03:52.176341+0000","last_became_peered":"2024-04-10T03:03:52.176341+0000","last_unstale":"2024-04-10T03:23:33.135889+0000","last_undegraded":"2024-04-10T03:23:33.135889+0000","last_fullsized":"2024-04-10T03:23:33.135889+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:43.066934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134756+0000","last_change":"2024-04-10T03:19:06.510620+0000","last_active":"2024-04-10T03:23:33.134756+0000","last_peered":"2024-04-10T03:23:33.134756+0000","last_clean":"2024-04-10T03:23:33.134756+0000","last_became_active":"2024-04-10T03:19:06.510274+0000","last_became_peered":"2024-04-10T03:19:06.510274+0000","last_unstale":"2024-04-10T03:23:33.134756+0000","last_undegraded":"2024-04-10T03:23:33.134756+0000","last_fullsized":"2024-04-10T03:23:33.134756+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:05:39.645055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139198+0000","last_change":"2024-04-10T03:22:58.819222+0000","last_active":"2024-04-10T03:23:33.139198+0000","last_peered":"2024-04-10T03:23:33.139198+0000","last_clean":"2024-04-10T03:23:33.139198+0000","last_became_active":"2024-04-10T03:22:58.818712+0000","last_became_peered":"2024-04-10T03:22:58.818712+0000","last_unstale":"2024-04-10T03:23:33.139198+0000","last_undegraded":"2024-04-10T03:23:33.139198+0000","last_fullsized":"2024-04-10T03:23:33.139198+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:09:03.181232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152117+0000","last_change":"2024-04-10T03:23:34.152117+0000","last_active":"2024-04-10T03:23:34.152117+0000","last_peered":"2024-04-10T03:23:34.152117+0000","last_clean":"2024-04-10T03:23:34.152117+0000","last_became_active":"2024-04-10T03:23:34.151730+0000","last_became_peered":"2024-04-10T03:23:34.151730+0000","last_unstale":"2024-04-10T03:23:34.152117+0000","last_undegraded":"2024-04-10T03:23:34.152117+0000","last_fullsized":"2024-04-10T03:23:34.152117+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:58:24.389948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152110+0000","last_change":"2024-04-10T03:23:34.150165+0000","last_active":"2024-04-10T03:23:34.152110+0000","last_peered":"2024-04-10T03:23:34.152110+0000","last_clean":"2024-04-10T03:23:34.152110+0000","last_became_active":"2024-04-10T03:23:34.149800+0000","last_became_peered":"2024-04-10T03:23:34.149800+0000","last_unstale":"2024-04-10T03:23:34.152110+0000","last_undegraded":"2024-04-10T03:23:34.152110+0000","last_fullsized":"2024-04-10T03:23:34.152110+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:39:23.070157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151191+0000","last_change":"2024-04-10T03:23:34.151191+0000","last_active":"2024-04-10T03:23:34.151191+0000","last_peered":"2024-04-10T03:23:34.151191+0000","last_clean":"2024-04-10T03:23:34.151191+0000","last_became_active":"2024-04-10T03:23:34.150769+0000","last_became_peered":"2024-04-10T03:23:34.150769+0000","last_unstale":"2024-04-10T03:23:34.151191+0000","last_undegraded":"2024-04-10T03:23:34.151191+0000","last_fullsized":"2024-04-10T03:23:34.151191+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:32:05.870647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143094+0000","last_change":"2024-04-10T03:03:52.171594+0000","last_active":"2024-04-10T03:23:33.143094+0000","last_peered":"2024-04-10T03:23:33.143094+0000","last_clean":"2024-04-10T03:23:33.143094+0000","last_became_active":"2024-04-10T03:03:52.171314+0000","last_became_peered":"2024-04-10T03:03:52.171314+0000","last_unstale":"2024-04-10T03:23:33.143094+0000","last_undegraded":"2024-04-10T03:23:33.143094+0000","last_fullsized":"2024-04-10T03:23:33.143094+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:37:55.120481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152376+0000","last_change":"2024-04-10T03:23:34.151952+0000","last_active":"2024-04-10T03:23:34.152376+0000","last_peered":"2024-04-10T03:23:34.152376+0000","last_clean":"2024-04-10T03:23:34.152376+0000","last_became_active":"2024-04-10T03:23:34.151635+0000","last_became_peered":"2024-04-10T03:23:34.151635+0000","last_unstale":"2024-04-10T03:23:34.152376+0000","last_undegraded":"2024-04-10T03:23:34.152376+0000","last_fullsized":"2024-04-10T03:23:34.152376+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:32:13.312002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":106,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.140340+0000","last_change":"2024-04-10T03:14:54.251381+0000","last_active":"2024-04-10T03:23:33.140340+0000","last_peered":"2024-04-10T03:23:33.140340+0000","last_clean":"2024-04-10T03:23:33.140340+0000","last_became_active":"2024-04-10T03:14:54.250834+0000","last_became_peered":"2024-04-10T03:14:54.250834+0000","last_unstale":"2024-04-10T03:23:33.140340+0000","last_undegraded":"2024-04-10T03:23:33.140340+0000","last_fullsized":"2024-04-10T03:23:33.140340+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:42:06.054546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151212+0000","last_change":"2024-04-10T03:23:34.151212+0000","last_active":"2024-04-10T03:23:34.151212+0000","last_peered":"2024-04-10T03:23:34.151212+0000","last_clean":"2024-04-10T03:23:34.151212+0000","last_became_active":"2024-04-10T03:23:34.150762+0000","last_became_peered":"2024-04-10T03:23:34.150762+0000","last_unstale":"2024-04-10T03:23:34.151212+0000","last_undegraded":"2024-04-10T03:23:34.151212+0000","last_fullsized":"2024-04-10T03:23:34.151212+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:21:33.466244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.141597+0000","last_change":"2024-04-10T03:14:54.256468+0000","last_active":"2024-04-10T03:23:33.141597+0000","last_peered":"2024-04-10T03:23:33.141597+0000","last_clean":"2024-04-10T03:23:33.141597+0000","last_became_active":"2024-04-10T03:14:54.255966+0000","last_became_peered":"2024-04-10T03:14:54.255966+0000","last_unstale":"2024-04-10T03:23:33.141597+0000","last_undegraded":"2024-04-10T03:23:33.141597+0000","last_fullsized":"2024-04-10T03:23:33.141597+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:33:52.775163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133944+0000","last_change":"2024-04-10T03:03:53.200529+0000","last_active":"2024-04-10T03:23:33.133944+0000","last_peered":"2024-04-10T03:23:33.133944+0000","last_clean":"2024-04-10T03:23:33.133944+0000","last_became_active":"2024-04-10T03:03:53.199080+0000","last_became_peered":"2024-04-10T03:03:53.199080+0000","last_unstale":"2024-04-10T03:23:33.133944+0000","last_undegraded":"2024-04-10T03:23:33.133944+0000","last_fullsized":"2024-04-10T03:23:33.133944+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T15:00:49.737785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143820+0000","last_change":"2024-04-10T03:19:06.503903+0000","last_active":"2024-04-10T03:23:33.143820+0000","last_peered":"2024-04-10T03:23:33.143820+0000","last_clean":"2024-04-10T03:23:33.143820+0000","last_became_active":"2024-04-10T03:19:06.503565+0000","last_became_peered":"2024-04-10T03:19:06.503565+0000","last_unstale":"2024-04-10T03:23:33.143820+0000","last_undegraded":"2024-04-10T03:23:33.143820+0000","last_fullsized":"2024-04-10T03:23:33.143820+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:04:44.209590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.150887+0000","last_change":"2024-04-10T03:23:34.150887+0000","last_active":"2024-04-10T03:23:34.150887+0000","last_peered":"2024-04-10T03:23:34.150887+0000","last_clean":"2024-04-10T03:23:34.150887+0000","last_became_active":"2024-04-10T03:23:34.150540+0000","last_became_peered":"2024-04-10T03:23:34.150540+0000","last_unstale":"2024-04-10T03:23:34.150887+0000","last_undegraded":"2024-04-10T03:23:34.150887+0000","last_fullsized":"2024-04-10T03:23:34.150887+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:24:21.573648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":82,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139127+0000","last_change":"2024-04-10T03:03:52.170665+0000","last_active":"2024-04-10T03:23:33.139127+0000","last_peered":"2024-04-10T03:23:33.139127+0000","last_clean":"2024-04-10T03:23:33.139127+0000","last_became_active":"2024-04-10T03:03:52.170385+0000","last_became_peered":"2024-04-10T03:03:52.170385+0000","last_unstale":"2024-04-10T03:23:33.139127+0000","last_undegraded":"2024-04-10T03:23:33.139127+0000","last_fullsized":"2024-04-10T03:23:33.139127+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:36:28.794484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.142580+0000","last_change":"2024-04-10T03:19:06.498472+0000","last_active":"2024-04-10T03:23:33.142580+0000","last_peered":"2024-04-10T03:23:33.142580+0000","last_clean":"2024-04-10T03:23:33.142580+0000","last_became_active":"2024-04-10T03:19:06.498130+0000","last_became_peered":"2024-04-10T03:19:06.498130+0000","last_unstale":"2024-04-10T03:23:33.142580+0000","last_undegraded":"2024-04-10T03:23:33.142580+0000","last_fullsized":"2024-04-10T03:23:33.142580+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:31:20.254590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153444+0000","last_change":"2024-04-10T03:23:34.153444+0000","last_active":"2024-04-10T03:23:34.153444+0000","last_peered":"2024-04-10T03:23:34.153444+0000","last_clean":"2024-04-10T03:23:34.153444+0000","last_became_active":"2024-04-10T03:23:34.153109+0000","last_became_peered":"2024-04-10T03:23:34.153109+0000","last_unstale":"2024-04-10T03:23:34.153444+0000","last_undegraded":"2024-04-10T03:23:34.153444+0000","last_fullsized":"2024-04-10T03:23:34.153444+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:23:44.895576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139198+0000","last_change":"2024-04-10T03:19:06.512811+0000","last_active":"2024-04-10T03:23:34.139198+0000","last_peered":"2024-04-10T03:23:34.139198+0000","last_clean":"2024-04-10T03:23:34.139198+0000","last_became_active":"2024-04-10T03:19:06.512465+0000","last_became_peered":"2024-04-10T03:19:06.512465+0000","last_unstale":"2024-04-10T03:23:34.139198+0000","last_undegraded":"2024-04-10T03:23:34.139198+0000","last_fullsized":"2024-04-10T03:23:34.139198+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:57:23.548014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":109,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.158338+0000","last_change":"2024-04-10T03:23:34.158338+0000","last_active":"2024-04-10T03:23:34.158338+0000","last_peered":"2024-04-10T03:23:34.158338+0000","last_clean":"2024-04-10T03:23:34.158338+0000","last_became_active":"2024-04-10T03:23:34.157304+0000","last_became_peered":"2024-04-10T03:23:34.157304+0000","last_unstale":"2024-04-10T03:23:34.158338+0000","last_undegraded":"2024-04-10T03:23:34.158338+0000","last_fullsized":"2024-04-10T03:23:34.158338+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T06:22:20.352365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.152209+0000","last_change":"2024-04-10T03:23:34.152209+0000","last_active":"2024-04-10T03:23:34.152209+0000","last_peered":"2024-04-10T03:23:34.152209+0000","last_clean":"2024-04-10T03:23:34.152209+0000","last_became_active":"2024-04-10T03:23:34.151856+0000","last_became_peered":"2024-04-10T03:23:34.151856+0000","last_unstale":"2024-04-10T03:23:34.152209+0000","last_undegraded":"2024-04-10T03:23:34.152209+0000","last_fullsized":"2024-04-10T03:23:34.152209+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:02:26.611265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.142812+0000","last_change":"2024-04-10T03:19:06.500320+0000","last_active":"2024-04-10T03:23:33.142812+0000","last_peered":"2024-04-10T03:23:33.142812+0000","last_clean":"2024-04-10T03:23:33.142812+0000","last_became_active":"2024-04-10T03:19:06.500001+0000","last_became_peered":"2024-04-10T03:19:06.500001+0000","last_unstale":"2024-04-10T03:23:33.142812+0000","last_undegraded":"2024-04-10T03:23:33.142812+0000","last_fullsized":"2024-04-10T03:23:33.142812+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:44:33.676068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.136333+0000","last_change":"2024-04-10T03:22:58.818406+0000","last_active":"2024-04-10T03:23:33.136333+0000","last_peered":"2024-04-10T03:23:33.136333+0000","last_clean":"2024-04-10T03:23:33.136333+0000","last_became_active":"2024-04-10T03:22:58.818060+0000","last_became_peered":"2024-04-10T03:22:58.818060+0000","last_unstale":"2024-04-10T03:23:33.136333+0000","last_undegraded":"2024-04-10T03:23:33.136333+0000","last_fullsized":"2024-04-10T03:23:33.136333+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:03:06.653280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.135105+0000","last_change":"2024-04-10T03:22:58.822444+0000","last_active":"2024-04-10T03:23:33.135105+0000","last_peered":"2024-04-10T03:23:33.135105+0000","last_clean":"2024-04-10T03:23:33.135105+0000","last_became_active":"2024-04-10T03:22:58.822064+0000","last_became_peered":"2024-04-10T03:22:58.822064+0000","last_unstale":"2024-04-10T03:23:33.135105+0000","last_undegraded":"2024-04-10T03:23:33.135105+0000","last_fullsized":"2024-04-10T03:23:33.135105+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:31:29.552226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.136960+0000","last_change":"2024-04-10T03:19:06.511142+0000","last_active":"2024-04-10T03:23:33.136960+0000","last_peered":"2024-04-10T03:23:33.136960+0000","last_clean":"2024-04-10T03:23:33.136960+0000","last_became_active":"2024-04-10T03:19:06.510840+0000","last_became_peered":"2024-04-10T03:19:06.510840+0000","last_unstale":"2024-04-10T03:23:33.136960+0000","last_undegraded":"2024-04-10T03:23:33.136960+0000","last_fullsized":"2024-04-10T03:23:33.136960+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:13:40.385441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151002+0000","last_change":"2024-04-10T03:23:34.151002+0000","last_active":"2024-04-10T03:23:34.151002+0000","last_peered":"2024-04-10T03:23:34.151002+0000","last_clean":"2024-04-10T03:23:34.151002+0000","last_became_active":"2024-04-10T03:23:34.150668+0000","last_became_peered":"2024-04-10T03:23:34.150668+0000","last_unstale":"2024-04-10T03:23:34.151002+0000","last_undegraded":"2024-04-10T03:23:34.151002+0000","last_fullsized":"2024-04-10T03:23:34.151002+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:24:55.948167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.149578+0000","last_change":"2024-04-10T03:23:34.148938+0000","last_active":"2024-04-10T03:23:34.149578+0000","last_peered":"2024-04-10T03:23:34.149578+0000","last_clean":"2024-04-10T03:23:34.149578+0000","last_became_active":"2024-04-10T03:23:34.148594+0000","last_became_peered":"2024-04-10T03:23:34.148594+0000","last_unstale":"2024-04-10T03:23:34.149578+0000","last_undegraded":"2024-04-10T03:23:34.149578+0000","last_fullsized":"2024-04-10T03:23:34.149578+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:54:24.394886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141335+0000","last_change":"2024-04-10T03:19:06.502575+0000","last_active":"2024-04-10T03:23:34.141335+0000","last_peered":"2024-04-10T03:23:34.141335+0000","last_clean":"2024-04-10T03:23:34.141335+0000","last_became_active":"2024-04-10T03:19:06.502292+0000","last_became_peered":"2024-04-10T03:19:06.502292+0000","last_unstale":"2024-04-10T03:23:34.141335+0000","last_undegraded":"2024-04-10T03:23:34.141335+0000","last_fullsized":"2024-04-10T03:23:34.141335+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T11:33:26.735169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151449+0000","last_change":"2024-04-10T03:23:34.151449+0000","last_active":"2024-04-10T03:23:34.151449+0000","last_peered":"2024-04-10T03:23:34.151449+0000","last_clean":"2024-04-10T03:23:34.151449+0000","last_became_active":"2024-04-10T03:23:34.151115+0000","last_became_peered":"2024-04-10T03:23:34.151115+0000","last_unstale":"2024-04-10T03:23:34.151449+0000","last_undegraded":"2024-04-10T03:23:34.151449+0000","last_fullsized":"2024-04-10T03:23:34.151449+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:34:25.299234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.140247+0000","last_change":"2024-04-10T03:19:06.508818+0000","last_active":"2024-04-10T03:23:34.140247+0000","last_peered":"2024-04-10T03:23:34.140247+0000","last_clean":"2024-04-10T03:23:34.140247+0000","last_became_active":"2024-04-10T03:19:06.507140+0000","last_became_peered":"2024-04-10T03:19:06.507140+0000","last_unstale":"2024-04-10T03:23:34.140247+0000","last_undegraded":"2024-04-10T03:23:34.140247+0000","last_fullsized":"2024-04-10T03:23:34.140247+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:00:54.482723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133932+0000","last_change":"2024-04-10T03:19:06.501152+0000","last_active":"2024-04-10T03:23:33.133932+0000","last_peered":"2024-04-10T03:23:33.133932+0000","last_clean":"2024-04-10T03:23:33.133932+0000","last_became_active":"2024-04-10T03:19:06.500732+0000","last_became_peered":"2024-04-10T03:19:06.500732+0000","last_unstale":"2024-04-10T03:23:33.133932+0000","last_undegraded":"2024-04-10T03:23:33.133932+0000","last_fullsized":"2024-04-10T03:23:33.133932+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:20:56.343876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134849+0000","last_change":"2024-04-10T03:14:54.260506+0000","last_active":"2024-04-10T03:23:33.134849+0000","last_peered":"2024-04-10T03:23:33.134849+0000","last_clean":"2024-04-10T03:23:33.134849+0000","last_became_active":"2024-04-10T03:14:54.260180+0000","last_became_peered":"2024-04-10T03:14:54.260180+0000","last_unstale":"2024-04-10T03:23:33.134849+0000","last_undegraded":"2024-04-10T03:23:33.134849+0000","last_fullsized":"2024-04-10T03:23:33.134849+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:38:08.636706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153406+0000","last_change":"2024-04-10T03:23:34.153406+0000","last_active":"2024-04-10T03:23:34.153406+0000","last_peered":"2024-04-10T03:23:34.153406+0000","last_clean":"2024-04-10T03:23:34.153406+0000","last_became_active":"2024-04-10T03:23:34.152999+0000","last_became_peered":"2024-04-10T03:23:34.152999+0000","last_unstale":"2024-04-10T03:23:34.153406+0000","last_undegraded":"2024-04-10T03:23:34.153406+0000","last_fullsized":"2024-04-10T03:23:34.153406+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:02:54.989305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":89,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.151258+0000","last_change":"2024-04-10T03:23:34.151258+0000","last_active":"2024-04-10T03:23:34.151258+0000","last_peered":"2024-04-10T03:23:34.151258+0000","last_clean":"2024-04-10T03:23:34.151258+0000","last_became_active":"2024-04-10T03:23:34.150937+0000","last_became_peered":"2024-04-10T03:23:34.150937+0000","last_unstale":"2024-04-10T03:23:34.151258+0000","last_undegraded":"2024-04-10T03:23:34.151258+0000","last_fullsized":"2024-04-10T03:23:34.151258+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:23:19.028348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139415+0000","last_change":"2024-04-10T03:22:58.820025+0000","last_active":"2024-04-10T03:23:34.139415+0000","last_peered":"2024-04-10T03:23:34.139415+0000","last_clean":"2024-04-10T03:23:34.139415+0000","last_became_active":"2024-04-10T03:22:58.819681+0000","last_became_peered":"2024-04-10T03:22:58.819681+0000","last_unstale":"2024-04-10T03:23:34.139415+0000","last_undegraded":"2024-04-10T03:23:34.139415+0000","last_fullsized":"2024-04-10T03:23:34.139415+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:02:29.026294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141439+0000","last_change":"2024-04-10T03:19:06.518701+0000","last_active":"2024-04-10T03:23:34.141439+0000","last_peered":"2024-04-10T03:23:34.141439+0000","last_clean":"2024-04-10T03:23:34.141439+0000","last_became_active":"2024-04-10T03:19:06.508984+0000","last_became_peered":"2024-04-10T03:19:06.508984+0000","last_unstale":"2024-04-10T03:23:34.141439+0000","last_undegraded":"2024-04-10T03:23:34.141439+0000","last_fullsized":"2024-04-10T03:23:34.141439+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:21:05.066857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134038+0000","last_change":"2024-04-10T03:22:58.818093+0000","last_active":"2024-04-10T03:23:33.134038+0000","last_peered":"2024-04-10T03:23:33.134038+0000","last_clean":"2024-04-10T03:23:33.134038+0000","last_became_active":"2024-04-10T03:22:58.817757+0000","last_became_peered":"2024-04-10T03:22:58.817757+0000","last_unstale":"2024-04-10T03:23:33.134038+0000","last_undegraded":"2024-04-10T03:23:33.134038+0000","last_fullsized":"2024-04-10T03:23:33.134038+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:26:03.034823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":478,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.138521+0000","last_change":"2024-04-10T03:14:54.256102+0000","last_active":"2024-04-10T03:23:33.138521+0000","last_peered":"2024-04-10T03:23:33.138521+0000","last_clean":"2024-04-10T03:23:33.138521+0000","last_became_active":"2024-04-10T03:14:54.255639+0000","last_became_peered":"2024-04-10T03:14:54.255639+0000","last_unstale":"2024-04-10T03:23:33.138521+0000","last_undegraded":"2024-04-10T03:23:33.138521+0000","last_fullsized":"2024-04-10T03:23:33.138521+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:00:04.753667+0000","last_clean_scrub_stamp":"2024-04-10T03:00:04.753667+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:55:00.311119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155980+0000","last_change":"2024-04-10T03:23:34.155980+0000","last_active":"2024-04-10T03:23:34.155980+0000","last_peered":"2024-04-10T03:23:34.155980+0000","last_clean":"2024-04-10T03:23:34.155980+0000","last_became_active":"2024-04-10T03:23:34.155532+0000","last_became_peered":"2024-04-10T03:23:34.155532+0000","last_unstale":"2024-04-10T03:23:34.155980+0000","last_undegraded":"2024-04-10T03:23:34.155980+0000","last_fullsized":"2024-04-10T03:23:34.155980+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:49:56.439170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139967+0000","last_change":"2024-04-10T03:03:53.201722+0000","last_active":"2024-04-10T03:23:33.139967+0000","last_peered":"2024-04-10T03:23:33.139967+0000","last_clean":"2024-04-10T03:23:33.139967+0000","last_became_active":"2024-04-10T03:03:53.201394+0000","last_became_peered":"2024-04-10T03:03:53.201394+0000","last_unstale":"2024-04-10T03:23:33.139967+0000","last_undegraded":"2024-04-10T03:23:33.139967+0000","last_fullsized":"2024-04-10T03:23:33.139967+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T04:53:16.107170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.141702+0000","last_change":"2024-04-10T03:19:06.502901+0000","last_active":"2024-04-10T03:23:33.141702+0000","last_peered":"2024-04-10T03:23:33.141702+0000","last_clean":"2024-04-10T03:23:33.141702+0000","last_became_active":"2024-04-10T03:19:06.502562+0000","last_became_peered":"2024-04-10T03:19:06.502562+0000","last_unstale":"2024-04-10T03:23:33.141702+0000","last_undegraded":"2024-04-10T03:23:33.141702+0000","last_fullsized":"2024-04-10T03:23:33.141702+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:42:35.746508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.154331+0000","last_change":"2024-04-10T03:23:34.154331+0000","last_active":"2024-04-10T03:23:34.154331+0000","last_peered":"2024-04-10T03:23:34.154331+0000","last_clean":"2024-04-10T03:23:34.154331+0000","last_became_active":"2024-04-10T03:23:34.153986+0000","last_became_peered":"2024-04-10T03:23:34.153986+0000","last_unstale":"2024-04-10T03:23:34.154331+0000","last_undegraded":"2024-04-10T03:23:34.154331+0000","last_fullsized":"2024-04-10T03:23:34.154331+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:17:50.203014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":141,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139445+0000","last_change":"2024-04-10T03:22:58.818486+0000","last_active":"2024-04-10T03:23:34.139445+0000","last_peered":"2024-04-10T03:23:34.139445+0000","last_clean":"2024-04-10T03:23:34.139445+0000","last_became_active":"2024-04-10T03:22:58.818170+0000","last_became_peered":"2024-04-10T03:22:58.818170+0000","last_unstale":"2024-04-10T03:23:34.139445+0000","last_undegraded":"2024-04-10T03:23:34.139445+0000","last_fullsized":"2024-04-10T03:23:34.139445+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:03:41.192593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.139271+0000","last_change":"2024-04-10T03:14:54.287221+0000","last_active":"2024-04-10T03:23:33.139271+0000","last_peered":"2024-04-10T03:23:33.139271+0000","last_clean":"2024-04-10T03:23:33.139271+0000","last_became_active":"2024-04-10T03:14:54.286933+0000","last_became_peered":"2024-04-10T03:14:54.286933+0000","last_unstale":"2024-04-10T03:23:33.139271+0000","last_undegraded":"2024-04-10T03:23:33.139271+0000","last_fullsized":"2024-04-10T03:23:33.139271+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:59:12.810453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.141757+0000","last_change":"2024-04-10T03:22:58.822527+0000","last_active":"2024-04-10T03:23:34.141757+0000","last_peered":"2024-04-10T03:23:34.141757+0000","last_clean":"2024-04-10T03:23:34.141757+0000","last_became_active":"2024-04-10T03:22:58.822195+0000","last_became_peered":"2024-04-10T03:22:58.822195+0000","last_unstale":"2024-04-10T03:23:34.141757+0000","last_undegraded":"2024-04-10T03:23:34.141757+0000","last_fullsized":"2024-04-10T03:23:34.141757+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:55:20.736706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":119,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.138878+0000","last_change":"2024-04-10T03:14:54.257574+0000","last_active":"2024-04-10T03:23:34.138878+0000","last_peered":"2024-04-10T03:23:34.138878+0000","last_clean":"2024-04-10T03:23:34.138878+0000","last_became_active":"2024-04-10T03:14:54.256568+0000","last_became_peered":"2024-04-10T03:14:54.256568+0000","last_unstale":"2024-04-10T03:23:34.138878+0000","last_undegraded":"2024-04-10T03:23:34.138878+0000","last_fullsized":"2024-04-10T03:23:34.138878+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:51:23.401981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":109,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156225+0000","last_change":"2024-04-10T03:23:34.156225+0000","last_active":"2024-04-10T03:23:34.156225+0000","last_peered":"2024-04-10T03:23:34.156225+0000","last_clean":"2024-04-10T03:23:34.156225+0000","last_became_active":"2024-04-10T03:23:34.155775+0000","last_became_peered":"2024-04-10T03:23:34.155775+0000","last_unstale":"2024-04-10T03:23:34.156225+0000","last_undegraded":"2024-04-10T03:23:34.156225+0000","last_fullsized":"2024-04-10T03:23:34.156225+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:47:53.125600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139985+0000","last_change":"2024-04-10T03:19:06.514780+0000","last_active":"2024-04-10T03:23:34.139985+0000","last_peered":"2024-04-10T03:23:34.139985+0000","last_clean":"2024-04-10T03:23:34.139985+0000","last_became_active":"2024-04-10T03:19:06.514387+0000","last_became_peered":"2024-04-10T03:19:06.514387+0000","last_unstale":"2024-04-10T03:23:34.139985+0000","last_undegraded":"2024-04-10T03:23:34.139985+0000","last_fullsized":"2024-04-10T03:23:34.139985+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:00:08.784016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156596+0000","last_change":"2024-04-10T03:23:34.156596+0000","last_active":"2024-04-10T03:23:34.156596+0000","last_peered":"2024-04-10T03:23:34.156596+0000","last_clean":"2024-04-10T03:23:34.156596+0000","last_became_active":"2024-04-10T03:23:34.156276+0000","last_became_peered":"2024-04-10T03:23:34.156276+0000","last_unstale":"2024-04-10T03:23:34.156596+0000","last_undegraded":"2024-04-10T03:23:34.156596+0000","last_fullsized":"2024-04-10T03:23:34.156596+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:54:35.146259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.143007+0000","last_change":"2024-04-10T03:22:58.819134+0000","last_active":"2024-04-10T03:23:33.143007+0000","last_peered":"2024-04-10T03:23:33.143007+0000","last_clean":"2024-04-10T03:23:33.143007+0000","last_became_active":"2024-04-10T03:22:58.818810+0000","last_became_peered":"2024-04-10T03:22:58.818810+0000","last_unstale":"2024-04-10T03:23:33.143007+0000","last_undegraded":"2024-04-10T03:23:33.143007+0000","last_fullsized":"2024-04-10T03:23:33.143007+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:02:37.121758+0000","last_clean_scrub_stamp":"2024-04-10T03:02:37.121758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:46:50.340812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139478+0000","last_change":"2024-04-10T03:19:06.512958+0000","last_active":"2024-04-10T03:23:34.139478+0000","last_peered":"2024-04-10T03:23:34.139478+0000","last_clean":"2024-04-10T03:23:34.139478+0000","last_became_active":"2024-04-10T03:19:06.512542+0000","last_became_peered":"2024-04-10T03:19:06.512542+0000","last_unstale":"2024-04-10T03:23:34.139478+0000","last_undegraded":"2024-04-10T03:23:34.139478+0000","last_fullsized":"2024-04-10T03:23:34.139478+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:21:09.385448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133994+0000","last_change":"2024-04-10T03:03:53.193384+0000","last_active":"2024-04-10T03:23:33.133994+0000","last_peered":"2024-04-10T03:23:33.133994+0000","last_clean":"2024-04-10T03:23:33.133994+0000","last_became_active":"2024-04-10T03:03:53.193048+0000","last_became_peered":"2024-04-10T03:03:53.193048+0000","last_unstale":"2024-04-10T03:23:33.133994+0000","last_undegraded":"2024-04-10T03:23:33.133994+0000","last_fullsized":"2024-04-10T03:23:33.133994+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T05:10:41.808720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":121,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139269+0000","last_change":"2024-04-10T03:14:54.259178+0000","last_active":"2024-04-10T03:23:34.139269+0000","last_peered":"2024-04-10T03:23:34.139269+0000","last_clean":"2024-04-10T03:23:34.139269+0000","last_became_active":"2024-04-10T03:14:54.257620+0000","last_became_peered":"2024-04-10T03:14:54.257620+0000","last_unstale":"2024-04-10T03:23:34.139269+0000","last_undegraded":"2024-04-10T03:23:34.139269+0000","last_fullsized":"2024-04-10T03:23:34.139269+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T07:23:43.049692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156329+0000","last_change":"2024-04-10T03:23:34.156329+0000","last_active":"2024-04-10T03:23:34.156329+0000","last_peered":"2024-04-10T03:23:34.156329+0000","last_clean":"2024-04-10T03:23:34.156329+0000","last_became_active":"2024-04-10T03:23:34.155926+0000","last_became_peered":"2024-04-10T03:23:34.155926+0000","last_unstale":"2024-04-10T03:23:34.156329+0000","last_undegraded":"2024-04-10T03:23:34.156329+0000","last_fullsized":"2024-04-10T03:23:34.156329+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:29:38.452599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":100,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134208+0000","last_change":"2024-04-10T03:19:06.501427+0000","last_active":"2024-04-10T03:23:33.134208+0000","last_peered":"2024-04-10T03:23:33.134208+0000","last_clean":"2024-04-10T03:23:33.134208+0000","last_became_active":"2024-04-10T03:19:06.500055+0000","last_became_peered":"2024-04-10T03:19:06.500055+0000","last_unstale":"2024-04-10T03:23:33.134208+0000","last_undegraded":"2024-04-10T03:23:33.134208+0000","last_fullsized":"2024-04-10T03:23:33.134208+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:11:52.273430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.153366+0000","last_change":"2024-04-10T03:23:34.153366+0000","last_active":"2024-04-10T03:23:34.153366+0000","last_peered":"2024-04-10T03:23:34.153366+0000","last_clean":"2024-04-10T03:23:34.153366+0000","last_became_active":"2024-04-10T03:23:34.152839+0000","last_became_peered":"2024-04-10T03:23:34.152839+0000","last_unstale":"2024-04-10T03:23:34.153366+0000","last_undegraded":"2024-04-10T03:23:34.153366+0000","last_fullsized":"2024-04-10T03:23:34.153366+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T14:17:50.987961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":83,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.134334+0000","last_change":"2024-04-10T03:03:52.169154+0000","last_active":"2024-04-10T03:23:33.134334+0000","last_peered":"2024-04-10T03:23:33.134334+0000","last_clean":"2024-04-10T03:23:33.134334+0000","last_became_active":"2024-04-10T03:03:52.168854+0000","last_became_peered":"2024-04-10T03:03:52.168854+0000","last_unstale":"2024-04-10T03:23:33.134334+0000","last_undegraded":"2024-04-10T03:23:33.134334+0000","last_fullsized":"2024-04-10T03:23:33.134334+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T09:01:21.091245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.148758+0000","last_change":"2024-04-10T03:23:34.148758+0000","last_active":"2024-04-10T03:23:34.148758+0000","last_peered":"2024-04-10T03:23:34.148758+0000","last_clean":"2024-04-10T03:23:34.148758+0000","last_became_active":"2024-04-10T03:23:34.148428+0000","last_became_peered":"2024-04-10T03:23:34.148428+0000","last_unstale":"2024-04-10T03:23:34.148758+0000","last_undegraded":"2024-04-10T03:23:34.148758+0000","last_fullsized":"2024-04-10T03:23:34.148758+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T12:41:21.082952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":107,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.155949+0000","last_change":"2024-04-10T03:23:34.155949+0000","last_active":"2024-04-10T03:23:34.155949+0000","last_peered":"2024-04-10T03:23:34.155949+0000","last_clean":"2024-04-10T03:23:34.155949+0000","last_became_active":"2024-04-10T03:23:34.155340+0000","last_became_peered":"2024-04-10T03:23:34.155340+0000","last_unstale":"2024-04-10T03:23:34.155949+0000","last_undegraded":"2024-04-10T03:23:34.155949+0000","last_fullsized":"2024-04-10T03:23:34.155949+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T10:59:23.588701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.156934+0000","last_change":"2024-04-10T03:23:34.156934+0000","last_active":"2024-04-10T03:23:34.156934+0000","last_peered":"2024-04-10T03:23:34.156934+0000","last_clean":"2024-04-10T03:23:34.156934+0000","last_became_active":"2024-04-10T03:23:34.156512+0000","last_became_peered":"2024-04-10T03:23:34.156512+0000","last_unstale":"2024-04-10T03:23:34.156934+0000","last_undegraded":"2024-04-10T03:23:34.156934+0000","last_fullsized":"2024-04-10T03:23:34.156934+0000","mapping_epoch":137,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T08:04:08.142967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":131,"reported_epoch":138,"state":"active+clean","last_fresh":"2024-04-10T03:23:34.139099+0000","last_change":"2024-04-10T03:22:58.819167+0000","last_active":"2024-04-10T03:23:34.139099+0000","last_peered":"2024-04-10T03:23:34.139099+0000","last_clean":"2024-04-10T03:23:34.139099+0000","last_became_active":"2024-04-10T03:22:58.818808+0000","last_became_peered":"2024-04-10T03:22:58.818808+0000","last_unstale":"2024-04-10T03:23:34.139099+0000","last_undegraded":"2024-04-10T03:23:34.139099+0000","last_fullsized":"2024-04-10T03:23:34.139099+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:51.144978+0000","last_clean_scrub_stamp":"2024-04-10T03:03:51.144978+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T13:45:04.307428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-04-10T03:23:33.133994+0000","last_change":"2024-04-10T03:19:06.511070+0000","last_active":"2024-04-10T03:23:33.133994+0000","last_peered":"2024-04-10T03:23:33.133994+0000","last_clean":"2024-04-10T03:23:33.133994+0000","last_became_active":"2024-04-10T03:19:06.510581+0000","last_became_peered":"2024-04-10T03:19:06.510581+0000","last_unstale":"2024-04-10T03:23:33.133994+0000","last_undegraded":"2024-04-10T03:23:33.133994+0000","last_fullsized":"2024-04-10T03:23:33.133994+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-10T03:03:52.153553+0000","last_clean_scrub_stamp":"2024-04-10T03:03:52.153553+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-11T03:10:59.440513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13613,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234257,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592028,"kb_used_data":2192,"kb_used_omap":1,"kb_used_meta":589822,"kb_avail":93153124,"statfs":{"total":95995035648,"available":95388798976,"internally_reserved":0,"allocated":2244608,"data_stored":1485199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":603978072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495895,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307728,"kb_used_data":2176,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93437424,"statfs":{"total":95995035648,"available":95679922176,"internally_reserved":0,"allocated":2228224,"data_stored":1484772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":114,"seq":489626271812,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":184712,"kb_used_data":1588,"kb_used_omap":5,"kb_used_meta":183098,"kb_avail":93560440,"statfs":{"total":95995035648,"available":95805890560,"internally_reserved":0,"allocated":1626112,"data_stored":890198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":187492631},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024054,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":282000,"kb_used_data":2176,"kb_used_omap":5,"kb_used_meta":279802,"kb_avail":93463152,"statfs":{"total":95995035648,"available":95706267648,"internally_reserved":0,"allocated":2228224,"data_stored":1481182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6053,"internal_metadata":286517339},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":132,"seq":566935683094,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":60304,"kb_used_data":1596,"kb_used_omap":6,"kb_used_meta":58681,"kb_avail":93684848,"statfs":{"total":95995035648,"available":95933284352,"internally_reserved":0,"allocated":1634304,"data_stored":890810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6680,"internal_metadata":60089832},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":137,"seq":588410519567,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78404,"kb_used_data":1592,"kb_used_omap":4,"kb_used_meta":76795,"kb_avail":93666748,"statfs":{"total":95995035648,"available":95914749952,"internally_reserved":0,"allocated":1630208,"data_stored":890288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4642,"internal_metadata":78638558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019168,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603732,"kb_used_data":1608,"kb_used_omap":1,"kb_used_meta":602110,"kb_avail":93141420,"statfs":{"total":95995035648,"available":95376814080,"internally_reserved":0,"allocated":1646592,"data_stored":894811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616560984},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757530,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896712,"kb_used_data":1596,"kb_used_omap":1,"kb_used_meta":895102,"kb_avail":92848440,"statfs":{"total":95995035648,"available":95076802560,"internally_reserved":0,"allocated":1634304,"data_stored":890818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":916584792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4349,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2165,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2938,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-10T03:24:38.297 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-10T03:24:38.297 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-10T03:24:38.297 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-10T03:24:38.297 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:24:38.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:38 smithi152 ceph-mon[88640]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:38.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:38 smithi152 ceph-mon[88640]: from='client.35433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-10T03:24:38.561 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:40.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:40 smithi102 ceph-mon[94535]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:40.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:40 smithi102 ceph-mon[98773]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:40.402 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:40.402 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:24:40.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:40 smithi152 ceph-mon[88640]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:41.177 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:40] "GET /metrics HTTP/1.1" 200 33968 "" "Prometheus/2.43.0" 2024-04-10T03:24:41.503 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:41 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/853987036' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:41.504 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:41 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/853987036' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:41.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:41 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/853987036' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:42.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:42 smithi152 ceph-mon[88640]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:42.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:42 smithi102 ceph-mon[94535]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:42.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:42 smithi102 ceph-mon[98773]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:44.082 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:24:44.354 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:44.476 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:44 smithi102 ceph-mon[94535]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:44.476 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:44 smithi102 ceph-mon[98773]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:44.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:44 smithi152 ceph-mon[88640]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:46.263 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:46.264 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:24:46.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:46 smithi152 ceph-mon[88640]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:46.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:46 smithi102 ceph-mon[94535]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:46.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:46 smithi102 ceph-mon[98773]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:47.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:47 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1001982072' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:47.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:47 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1001982072' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:47.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:47 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1001982072' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:48.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:48 smithi152 ceph-mon[88640]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:48.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:48 smithi102 ceph-mon[94535]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:48.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:48.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:48 smithi102 ceph-mon[98773]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:48.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:24:49.950 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:24:50.222 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:50.294 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:50 smithi102 ceph-mon[94535]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:50.294 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:50 smithi102 ceph-mon[98773]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:50.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:50 smithi152 ceph-mon[88640]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:50.985 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:24:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:24:50] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:24:52.107 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:52.107 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:24:52.401 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:52 smithi102 ceph-mon[94535]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:52.402 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2977742459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:52.402 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:52 smithi102 ceph-mon[98773]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:52.402 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2977742459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:52.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:52 smithi152 ceph-mon[88640]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:52.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2977742459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:53.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:53 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:24:53.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:53 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:24:53.541 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:53 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:24:54.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:54 smithi102 ceph-mon[94535]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:54.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:54 smithi102 ceph-mon[98773]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:54.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:54 smithi152 ceph-mon[88640]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:55.755 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:24:56.101 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:24:56.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:56 smithi152 ceph-mon[88640]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:56.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:56 smithi102 ceph-mon[94535]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:56.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:56 smithi102 ceph-mon[98773]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:57 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:57 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:57 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:57 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:57 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.618 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:24:58.618 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:24:58.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[94535]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:58.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:24:58.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:24:58.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4024855559' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:58.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[98773]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:58.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:24:58.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:24:58.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:58.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:58 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4024855559' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:24:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:58 smithi152 ceph-mon[88640]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:24:59.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:24:59.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:24:59.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:24:59.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:58 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4024855559' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:59 smithi152 ceph-mon[88640]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:00.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:24:59 smithi152 ceph-mon[88640]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:00.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:59 smithi102 ceph-mon[94535]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:00.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:24:59 smithi102 ceph-mon[94535]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:59 smithi102 ceph-mon[98773]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:24:59 smithi102 ceph-mon[98773]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:01.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:00] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:25:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:01 smithi152 ceph-mon[88640]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:02.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:01 smithi102 ceph-mon[94535]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:02.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:01 smithi102 ceph-mon[98773]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:02.268 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:02.536 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:03.980 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:03 smithi102 ceph-mon[94535]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:03.980 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:03.980 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:03 smithi102 ceph-mon[98773]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:03.980 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:03 smithi152 ceph-mon[88640]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:04.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:04.453 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:04.454 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:05.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:04 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/404663562' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:05.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:04 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/404663562' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:05.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:04 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/404663562' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:06.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:05 smithi102 ceph-mon[94535]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:06.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:05 smithi102 ceph-mon[98773]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:06.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:05 smithi152 ceph-mon[88640]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:08.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:07 smithi102 ceph-mon[94535]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:07 smithi102 ceph-mon[98773]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:08.063 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:07 smithi152 ceph-mon[88640]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:08.162 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:08.428 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:09.941 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:09 smithi102 ceph-mon[94535]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:09.942 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:09 smithi102 ceph-mon[98773]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:10.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:09 smithi152 ceph-mon[88640]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:10.551 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:10.551 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:10.836 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:10 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3092857983' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:10.836 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:10 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3092857983' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:11.219 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:10] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:25:11.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:10 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3092857983' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:12.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:11 smithi102 ceph-mon[94535]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:12.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:11 smithi102 ceph-mon[98773]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:12.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:11 smithi152 ceph-mon[88640]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:14.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:13 smithi102 ceph-mon[94535]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:14.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:13 smithi102 ceph-mon[98773]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:14.245 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:14.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:13 smithi152 ceph-mon[88640]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:14.511 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:15.951 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:15 smithi102 ceph-mon[94535]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:15.951 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:15 smithi102 ceph-mon[98773]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:16.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:15 smithi152 ceph-mon[88640]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:16.446 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:16.446 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:17.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:16 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2344612964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:17.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:16 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2344612964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:17.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:16 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2344612964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:18.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:17 smithi152 ceph-mon[88640]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:18.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:17 smithi102 ceph-mon[94535]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:18.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:17 smithi102 ceph-mon[98773]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:19.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:19.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:19.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:20.127 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:20.158 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:19 smithi102 ceph-mon[94535]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:20.159 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:19 smithi102 ceph-mon[98773]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:20.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:19 smithi152 ceph-mon[88640]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:20.400 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:21.041 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:20] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:25:22.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:21 smithi152 ceph-mon[88640]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:22.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:21 smithi102 ceph-mon[94535]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:22.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:21 smithi102 ceph-mon[98773]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:22.293 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:22.293 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:22.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:22 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1268217025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:22.942 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:22 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1268217025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:23.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:22 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1268217025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:23 smithi152 ceph-mon[88640]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:24.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:23 smithi102 ceph-mon[94535]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:24.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:23 smithi102 ceph-mon[98773]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:25.968 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:26.237 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:26.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:25 smithi152 ceph-mon[88640]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:26.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:25 smithi102 ceph-mon[94535]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:26.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:25 smithi102 ceph-mon[98773]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:27 smithi102 ceph-mon[98773]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:27 smithi102 ceph-mon[94535]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:28.089 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:28.089 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:28.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:27 smithi152 ceph-mon[88640]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:29.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:28 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3735966864' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:29.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:28 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3735966864' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:29.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:28 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3735966864' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:30.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:29 smithi152 ceph-mon[88640]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:30.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:29 smithi102 ceph-mon[94535]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:30.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:29 smithi102 ceph-mon[98773]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:31.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:30] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:25:31.720 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:32.009 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:32.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:31 smithi102 ceph-mon[94535]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:32.041 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:31 smithi102 ceph-mon[98773]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:32.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:31 smithi152 ceph-mon[88640]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:33.983 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:33.983 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:33.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:33 smithi102 ceph-mon[94535]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:33.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:33.984 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:33 smithi102 ceph-mon[98773]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:33.984 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:33 smithi152 ceph-mon[88640]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:35.041 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:34 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1535122300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:35.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:34 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1535122300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:35.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:34 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1535122300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:36.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:35 smithi152 ceph-mon[88640]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:36.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:35 smithi102 ceph-mon[94535]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:36.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:35 smithi102 ceph-mon[98773]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:37.617 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:37.889 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:38.011 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:37 smithi102 ceph-mon[94535]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:38.011 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:37 smithi102 ceph-mon[98773]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:38.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:37 smithi152 ceph-mon[88640]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:39.754 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:39.754 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:39 smithi102 ceph-mon[94535]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:39 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2587979965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:39 smithi102 ceph-mon[98773]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:40.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:39 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2587979965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:40.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:39 smithi152 ceph-mon[88640]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:40.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:39 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2587979965' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:41.291 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:40] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.43.0" 2024-04-10T03:25:42.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:41 smithi102 ceph-mon[94535]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:42.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:41 smithi102 ceph-mon[98773]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:42.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:41 smithi152 ceph-mon[88640]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:43.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:42 smithi102 ceph-mon[94535]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:43.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:42 smithi102 ceph-mon[98773]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:43.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:42 smithi152 ceph-mon[88640]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:43.478 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:43.749 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:45.213 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:45 smithi102 ceph-mon[94535]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:45.213 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:45 smithi102 ceph-mon[98773]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:45.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:45 smithi152 ceph-mon[88640]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:45.687 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:45.687 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:46.288 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:46 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1280514261' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:46.288 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:46 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1280514261' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:46.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:46 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1280514261' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:47.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:47 smithi102 ceph-mon[94535]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:47.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:47 smithi102 ceph-mon[98773]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:47.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:47 smithi152 ceph-mon[88640]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:48.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:48.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:48.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:25:49.315 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:49.345 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:49 smithi102 ceph-mon[94535]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:49.345 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:49 smithi102 ceph-mon[98773]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:49.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:49 smithi152 ceph-mon[88640]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:49.579 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:50.990 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:25:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:25:50] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-04-10T03:25:51.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:51 smithi102 ceph-mon[94535]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:51 smithi102 ceph-mon[98773]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:51.468 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:51.469 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:51.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:51 smithi152 ceph-mon[88640]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:52.162 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:52 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/91724729' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:52.162 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:52 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/91724729' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:52.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:52 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/91724729' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:53.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:53 smithi152 ceph-mon[88640]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:53.541 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:53 smithi102 ceph-mon[94535]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:53.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:53 smithi102 ceph-mon[98773]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:55.195 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:25:55.461 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:25:55.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:55 smithi152 ceph-mon[88640]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:55.542 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:55 smithi102 ceph-mon[94535]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:55.542 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:55 smithi102 ceph-mon[98773]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:57.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:57 smithi102 ceph-mon[94535]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:57.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:57 smithi102 ceph-mon[98773]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:57.313 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:25:57.314 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:25:57.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:57 smithi152 ceph-mon[88640]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:25:58.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:58 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/940998359' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:58.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:58 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:25:58.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:58 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/940998359' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:58.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:58 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:25:58.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:58 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/940998359' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:25:58.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:58 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:26:00.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:25:59 smithi152 ceph-mon[88640]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:00.025 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:25:59 smithi102 ceph-mon[98773]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:00.025 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:25:59 smithi102 ceph-mon[94535]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:00.974 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:01.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:00] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-04-10T03:26:01.834 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:01 smithi152 ceph-mon[88640]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:02.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:01 smithi102 ceph-mon[94535]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:02.021 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:01 smithi102 ceph-mon[98773]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.019 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.020 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.021 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:26:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:03.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:26:03.769 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:03.770 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:03 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3595612926' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:03 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3595612926' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:04.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:04.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:03 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3595612926' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:05.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:04 smithi152 ceph-mon[88640]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:05.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:04 smithi102 ceph-mon[94535]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:05.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:04 smithi102 ceph-mon[98773]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:07.446 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:07.709 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:07 smithi102 ceph-mon[94535]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:08.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:07 smithi102 ceph-mon[98773]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:08.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:07 smithi152 ceph-mon[88640]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:09.621 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:09.621 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:09.976 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:09 smithi102 ceph-mon[94535]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:09.976 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:09 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2691526755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:09.976 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:09 smithi102 ceph-mon[98773]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:09.976 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:09 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2691526755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:10.064 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:09 smithi152 ceph-mon[88640]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:10.064 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:09 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2691526755' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:11.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:10] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:26:11.829 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:11 smithi102 ceph-mon[94535]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:11.829 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:11 smithi102 ceph-mon[98773]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:12.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:11 smithi152 ceph-mon[88640]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:13.310 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:13.577 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:13.964 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:13 smithi102 ceph-mon[94535]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:13.964 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:13 smithi102 ceph-mon[98773]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:13 smithi152 ceph-mon[88640]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:15.556 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:15.556 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:15.926 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:15 smithi102 ceph-mon[94535]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:15.926 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:15 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3973562097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:15.926 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:15 smithi102 ceph-mon[98773]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:15.926 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:15 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3973562097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:16.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:15 smithi152 ceph-mon[88640]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:16.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:15 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3973562097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:18.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:17 smithi102 ceph-mon[94535]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:17 smithi102 ceph-mon[98773]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:17 smithi152 ceph-mon[88640]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:19.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:19.243 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:19.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:19.515 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:19.887 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:19 smithi102 ceph-mon[94535]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:19.887 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:19 smithi102 ceph-mon[98773]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:20.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:19 smithi152 ceph-mon[88640]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:21.287 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:20] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:26:21.630 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:21.630 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:21.982 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:21 smithi102 ceph-mon[98773]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:21.982 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:21 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2460260428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:21.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:21 smithi102 ceph-mon[94535]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:21.983 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:21 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2460260428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:22.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:21 smithi152 ceph-mon[88640]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:22.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:21 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2460260428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:23 smithi102 ceph-mon[94535]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:24.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:23 smithi102 ceph-mon[98773]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:23 smithi152 ceph-mon[88640]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:25.308 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:25.578 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:26.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:25 smithi102 ceph-mon[94535]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:26.028 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:25 smithi102 ceph-mon[98773]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:26.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:25 smithi152 ceph-mon[88640]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:27.570 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:27.570 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:28.208 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:27 smithi102 ceph-mon[94535]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:28.208 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:27 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/784376853' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:28.208 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:27 smithi102 ceph-mon[98773]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:28.208 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:27 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/784376853' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:28.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:27 smithi152 ceph-mon[88640]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:28.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:27 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/784376853' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:30.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:29 smithi152 ceph-mon[88640]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:30.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:29 smithi102 ceph-mon[94535]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:30.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:29 smithi102 ceph-mon[98773]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:31.234 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:31.264 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:30] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:26:31.505 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:31.911 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:31 smithi102 ceph-mon[98773]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:31.911 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:31 smithi102 ceph-mon[94535]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:32.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:31 smithi152 ceph-mon[88640]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:33.454 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:33.454 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:34.082 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[98773]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:34.082 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:34.082 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3692983403' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:34.083 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[94535]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:34.083 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:34.083 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3692983403' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:33 smithi152 ceph-mon[88640]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3692983403' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:36.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:35 smithi152 ceph-mon[88640]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:36.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:35 smithi102 ceph-mon[94535]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:36.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:35 smithi102 ceph-mon[98773]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:37.107 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:37.378 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:38.235 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:37 smithi102 ceph-mon[94535]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:38.235 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:37 smithi102 ceph-mon[98773]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:38.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:37 smithi152 ceph-mon[88640]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:39.293 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:39.293 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:39.955 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:39 smithi102 ceph-mon[94535]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:39.956 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:39 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/205878553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:39.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:39 smithi102 ceph-mon[98773]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:39.956 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:39 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/205878553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:40.064 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:39 smithi152 ceph-mon[88640]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:40.064 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:39 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/205878553' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:41.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:40] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.43.0" 2024-04-10T03:26:42.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:41 smithi152 ceph-mon[88640]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:42.291 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:41 smithi102 ceph-mon[94535]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:42.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:41 smithi102 ceph-mon[98773]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:42.981 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:43.243 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:44.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:43 smithi102 ceph-mon[98773]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:44.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:43 smithi102 ceph-mon[94535]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:44.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:43 smithi152 ceph-mon[88640]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:45.135 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:45.136 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:46.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:45 smithi152 ceph-mon[88640]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:46.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:45 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3955506632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:45 smithi102 ceph-mon[94535]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:45 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3955506632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:46.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:45 smithi102 ceph-mon[98773]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:46.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:45 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3955506632' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:48.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:47 smithi152 ceph-mon[88640]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:48.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:47 smithi102 ceph-mon[94535]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:48.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:47 smithi102 ceph-mon[98773]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:48.823 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:49.087 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:49.209 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:49.210 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:49.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:26:50.136 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:49 smithi102 ceph-mon[94535]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:50.136 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:49 smithi102 ceph-mon[98773]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:50.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:49 smithi152 ceph-mon[88640]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:50.904 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:50.904 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:51.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:26:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:26:50] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:26:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:51 smithi152 ceph-mon[88640]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:52.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:51 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2517688249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:52.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:51 smithi102 ceph-mon[94535]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:52.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:51 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2517688249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:52.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:51 smithi102 ceph-mon[98773]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:52.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:51 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2517688249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:54.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:53 smithi152 ceph-mon[88640]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:54.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:53 smithi102 ceph-mon[94535]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:54.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:53 smithi102 ceph-mon[98773]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:54.521 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:26:54.783 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:26:56.162 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:55 smithi102 ceph-mon[94535]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:56.163 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:55 smithi102 ceph-mon[98773]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:56.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:55 smithi152 ceph-mon[88640]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:56.663 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:26:56.663 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:26:57.008 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3216926917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:57.008 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3216926917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:57.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3216926917' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:26:58.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:57 smithi152 ceph-mon[88640]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:58.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:57 smithi102 ceph-mon[94535]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:26:58.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:57 smithi102 ceph-mon[98773]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:00.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:26:59 smithi152 ceph-mon[88640]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:00.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:26:59 smithi102 ceph-mon[94535]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:00.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:26:59 smithi102 ceph-mon[98773]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:00.309 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:00.577 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:00.954 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:00] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:27:01.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:01 smithi102 ceph-mon[94535]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:01.982 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:01 smithi102 ceph-mon[98773]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:02.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:01 smithi152 ceph-mon[88640]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:02.472 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:02.473 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:03.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3775067494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:03.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:02 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:27:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3775067494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:02 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:27:03.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3775067494' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:03.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:02 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:27:04.019 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:03 smithi152 ceph-mon[88640]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:04.019 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:03 smithi102 ceph-mon[94535]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:03 smithi102 ceph-mon[98773]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:05.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:04 smithi102 ceph-mon[94535]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:05.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:04 smithi102 ceph-mon[98773]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:05.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:04 smithi152 ceph-mon[88640]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:06.235 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:06.602 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[98773]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[94535]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:07.691 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:07 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:07 smithi152 ceph-mon[88640]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:07 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.461 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:08.461 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3121057678' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:27:08.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:08.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:08 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3121057678' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:08 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:27:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:08 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:27:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:08 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:27:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:08 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3121057678' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:09 smithi152 ceph-mon[88640]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:09 smithi152 ceph-mon[88640]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:09 smithi102 ceph-mon[94535]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:09 smithi102 ceph-mon[94535]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:09 smithi102 ceph-mon[98773]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:10.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:09 smithi102 ceph-mon[98773]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:11.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:10] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.43.0" 2024-04-10T03:27:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:11 smithi152 ceph-mon[88640]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:11 smithi102 ceph-mon[94535]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:11 smithi102 ceph-mon[98773]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:12.166 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:12.432 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:13.929 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:13 smithi102 ceph-mon[94535]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:13.930 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:13 smithi102 ceph-mon[98773]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:14.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:13 smithi152 ceph-mon[88640]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:14.454 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:14.454 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:14.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4192608175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:14.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4192608175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4192608175' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:16.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:15 smithi152 ceph-mon[88640]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:15 smithi102 ceph-mon[94535]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:16.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:15 smithi102 ceph-mon[98773]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:18.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:17 smithi152 ceph-mon[88640]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:18.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:17 smithi102 ceph-mon[94535]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:18.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:17 smithi102 ceph-mon[98773]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:18.124 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:18.395 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:18.784 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:18.785 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:19.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:19.787 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:19 smithi102 ceph-mon[94535]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:19.788 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:19 smithi102 ceph-mon[98773]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:20.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:19 smithi152 ceph-mon[88640]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:20.293 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:20.293 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:20.896 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2354833014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:20.896 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2354833014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:21.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2354833014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:21.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:20] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:27:22.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:21 smithi152 ceph-mon[88640]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:22.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:21 smithi102 ceph-mon[94535]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:22.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:21 smithi102 ceph-mon[98773]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:23.923 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:23.953 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:23 smithi102 ceph-mon[94535]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:23.953 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:23 smithi102 ceph-mon[98773]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:24.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:23 smithi152 ceph-mon[88640]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:24.188 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:25.815 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:25 smithi102 ceph-mon[94535]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:25.815 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:25 smithi102 ceph-mon[98773]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:26.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:25 smithi152 ceph-mon[88640]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:26.057 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:26.057 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:27.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1647775988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:27.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1647775988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1647775988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:28.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:27 smithi152 ceph-mon[88640]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:28.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:27 smithi102 ceph-mon[94535]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:27 smithi102 ceph-mon[98773]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:29.669 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:29.943 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:30.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:29 smithi152 ceph-mon[88640]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:30.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:29 smithi102 ceph-mon[98773]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:30.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:29 smithi102 ceph-mon[94535]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:31.200 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:30] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:27:31.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:31 smithi102 ceph-mon[94535]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:31.792 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:31 smithi102 ceph-mon[98773]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:31.793 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:31.794 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:32.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:31 smithi152 ceph-mon[88640]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:33.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:32 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3137148048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:33.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:32 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3137148048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:33.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:32 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3137148048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:33 smithi152 ceph-mon[88640]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:34.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:33 smithi102 ceph-mon[98773]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:34.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:33 smithi102 ceph-mon[94535]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:35.598 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:35.871 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:35.993 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:35 smithi102 ceph-mon[94535]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:35.993 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:35 smithi102 ceph-mon[98773]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:36.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:35 smithi152 ceph-mon[88640]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:37.801 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:37.801 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:38.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:37 smithi152 ceph-mon[88640]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:38.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:37 smithi102 ceph-mon[94535]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:38.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:37 smithi102 ceph-mon[98773]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:38.782 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:38 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4192349970' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:38.782 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:38 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4192349970' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:39.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:38 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4192349970' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:40.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:39 smithi152 ceph-mon[88640]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:39 smithi102 ceph-mon[94535]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:40.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:39 smithi102 ceph-mon[98773]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:41.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:40] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:27:41.518 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:41.787 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:41.909 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:41 smithi102 ceph-mon[94535]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:41.909 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:41 smithi102 ceph-mon[98773]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:42.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:41 smithi152 ceph-mon[88640]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:43.729 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:43.729 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:43.729 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:43 smithi102 ceph-mon[94535]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:43.729 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:43 smithi102 ceph-mon[98773]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:43 smithi152 ceph-mon[88640]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:45.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:44 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/543418777' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:45.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:44 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/543418777' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:45.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:44 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/543418777' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:46.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:45 smithi152 ceph-mon[88640]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:46.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:45 smithi102 ceph-mon[94535]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:46.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:45 smithi102 ceph-mon[98773]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:47.627 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:47.905 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:48.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:47 smithi152 ceph-mon[88640]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:48.028 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:47 smithi102 ceph-mon[94535]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:48.028 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:47 smithi102 ceph-mon[98773]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:49.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:48 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:49.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:48 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:49.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:48 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:27:49.941 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:49.941 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:49.942 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:49 smithi102 ceph-mon[94535]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:49.942 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:49 smithi102 ceph-mon[98773]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:50.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:49 smithi152 ceph-mon[88640]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:50.934 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:50 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1733089385' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:50.934 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:50 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1733089385' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:51.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:50 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1733089385' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:51.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:27:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:27:50] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:27:52.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:51 smithi152 ceph-mon[88640]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:52.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:51 smithi102 ceph-mon[94535]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:52.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:51 smithi102 ceph-mon[98773]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:53.633 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:53.900 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:27:54.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:53 smithi152 ceph-mon[88640]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:54.022 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:53 smithi102 ceph-mon[94535]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:54.023 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:53 smithi102 ceph-mon[98773]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:55.857 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:27:55.857 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:27:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:55 smithi152 ceph-mon[88640]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:56.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:55 smithi102 ceph-mon[94535]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:56.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:55 smithi102 ceph-mon[98773]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:57.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4056528576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:57.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4056528576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:57.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4056528576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:27:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:57 smithi152 ceph-mon[88640]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:57 smithi102 ceph-mon[94535]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:58.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:57 smithi102 ceph-mon[98773]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:27:59.656 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:27:59.921 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:00.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:27:59 smithi152 ceph-mon[88640]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:27:59 smithi102 ceph-mon[98773]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:00.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:27:59 smithi102 ceph-mon[94535]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:01.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:00] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:28:01.860 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:01.860 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:01 smithi152 ceph-mon[88640]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:02.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:01 smithi102 ceph-mon[94535]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:02.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:01 smithi102 ceph-mon[98773]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/702728636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:03.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/702728636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/702728636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:03 smithi152 ceph-mon[88640]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:03 smithi102 ceph-mon[94535]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:04.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:03 smithi102 ceph-mon[98773]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:05.518 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:05.792 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:05.915 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:05 smithi102 ceph-mon[94535]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:05.915 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:05 smithi102 ceph-mon[98773]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:05 smithi152 ceph-mon[88640]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:07.732 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:07.732 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:07 smithi152 ceph-mon[88640]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:08.039 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:07 smithi102 ceph-mon[98773]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:08.039 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:07 smithi102 ceph-mon[94535]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:08.989 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:08 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2716547516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:08.990 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:08 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:28:08.990 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:08 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2716547516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:08.990 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:08 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:28:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:08 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2716547516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:08 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:28:09.992 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:09 smithi102 ceph-mon[94535]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:09.992 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:09 smithi102 ceph-mon[98773]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:09 smithi152 ceph-mon[88640]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:11.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:10] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.43.0" 2024-04-10T03:28:11.356 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:11.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:11 smithi152 ceph-mon[88640]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:11.850 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:11 smithi102 ceph-mon[94535]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:11 smithi102 ceph-mon[98773]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:12.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.943 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:28:12.944 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:28:13.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:28:13.720 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:13.720 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:14.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[94535]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[94535]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[94535]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1549524137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[98773]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[98773]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[98773]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:13 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1549524137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:13 smithi152 ceph-mon[88640]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:13 smithi152 ceph-mon[88640]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:13 smithi152 ceph-mon[88640]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:14.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:13 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1549524137' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:16.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:15 smithi152 ceph-mon[88640]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:16.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:15 smithi102 ceph-mon[98773]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:16.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:15 smithi102 ceph-mon[94535]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:17.531 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:17.792 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:18.072 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:17 smithi102 ceph-mon[94535]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:18.072 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:17 smithi102 ceph-mon[98773]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:17 smithi152 ceph-mon[88640]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:19.194 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:19.194 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:19.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:19.816 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:19.816 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:20.143 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:19 smithi102 ceph-mon[94535]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:20.143 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:19 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/858555656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:20.143 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:19 smithi102 ceph-mon[98773]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:20.143 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:19 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/858555656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:20.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:19 smithi152 ceph-mon[88640]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:20.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:19 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/858555656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:21.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:20] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.43.0" 2024-04-10T03:28:22.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:21 smithi152 ceph-mon[88640]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:22.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:21 smithi102 ceph-mon[94535]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:22.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:21 smithi102 ceph-mon[98773]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:23.627 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:23.891 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:24.013 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:23 smithi102 ceph-mon[94535]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:24.013 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:23 smithi102 ceph-mon[98773]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:23 smithi152 ceph-mon[88640]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:25.882 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:25.882 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:25.882 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:25 smithi102 ceph-mon[94535]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:25.882 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:25 smithi102 ceph-mon[98773]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:26.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:25 smithi152 ceph-mon[88640]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:27.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1893528922' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:27.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1893528922' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:27.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1893528922' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:28.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:27 smithi152 ceph-mon[88640]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:28.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:27 smithi102 ceph-mon[94535]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:28.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:27 smithi102 ceph-mon[98773]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:29.500 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:29.765 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:30.136 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:29 smithi102 ceph-mon[94535]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:30.137 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:29 smithi102 ceph-mon[98773]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:30.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:29 smithi152 ceph-mon[88640]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:31.262 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:30] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.43.0" 2024-04-10T03:28:31.750 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:31.750 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:32.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:31 smithi102 ceph-mon[94535]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:32.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:31 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3649058931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:32.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:31 smithi102 ceph-mon[98773]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:32.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:31 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3649058931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:32.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:31 smithi152 ceph-mon[88640]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:32.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:31 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3649058931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:33 smithi152 ceph-mon[88640]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:34.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:33 smithi102 ceph-mon[94535]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:34.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:34.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:33 smithi102 ceph-mon[98773]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:34.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:35.485 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:35.753 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:36.145 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:35 smithi102 ceph-mon[94535]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:36.145 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:35 smithi102 ceph-mon[98773]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:36.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:35 smithi152 ceph-mon[88640]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:37.782 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:37.782 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:38.091 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:37 smithi102 ceph-mon[94535]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:38.091 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:37 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1499718245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:38.091 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:37 smithi102 ceph-mon[98773]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:38.091 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:37 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1499718245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:38.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:37 smithi152 ceph-mon[88640]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:38.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:37 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1499718245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:40.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:39 smithi102 ceph-mon[94535]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:40.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:39 smithi102 ceph-mon[98773]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:40.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:39 smithi152 ceph-mon[88640]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:41.292 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:40] "GET /metrics HTTP/1.1" 200 33969 "" "Prometheus/2.43.0" 2024-04-10T03:28:41.500 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:41.777 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:42.145 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:41 smithi102 ceph-mon[98773]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:42.146 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:41 smithi102 ceph-mon[94535]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:42.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:41 smithi152 ceph-mon[88640]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:43.985 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:43.985 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:43.986 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:43 smithi102 ceph-mon[94535]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:43.986 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:43 smithi102 ceph-mon[98773]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:44.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:43 smithi152 ceph-mon[88640]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:45.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:44 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/4270299157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:45.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:44 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/4270299157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:45.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:44 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/4270299157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:46.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:45 smithi152 ceph-mon[88640]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:46.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:45 smithi102 ceph-mon[94535]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:46.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:45 smithi102 ceph-mon[98773]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:47.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:46 smithi152 ceph-mon[88640]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:47.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:46 smithi102 ceph-mon[94535]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:47.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:46 smithi102 ceph-mon[98773]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:47.699 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:47.978 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:48.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:47 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:48.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:47 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:48.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:47 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:28:49.156 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:48 smithi102 ceph-mon[94535]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:49.156 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:48 smithi102 ceph-mon[98773]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:49.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:48 smithi152 ceph-mon[88640]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:49.918 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:49.918 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:50.239 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:49 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3754024255' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:50.239 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:49 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3754024255' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:50.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:49 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3754024255' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:51.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:50 smithi152 ceph-mon[88640]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:51.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:50 smithi102 ceph-mon[98773]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:51.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:50 smithi102 ceph-mon[94535]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:51.293 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:28:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:28:50] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:28:53.666 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:28:53.938 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:28:54.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:53 smithi152 ceph-mon[88640]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:54.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:53 smithi102 ceph-mon[94535]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:54.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:53 smithi102 ceph-mon[98773]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:55.990 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:28:55.990 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:28:55.991 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:55 smithi102 ceph-mon[94535]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:55.991 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:55 smithi102 ceph-mon[98773]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:56.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:55 smithi152 ceph-mon[88640]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:57.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:56 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2801674787' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:57.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:56 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2801674787' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:57.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:56 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2801674787' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:28:58.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:57 smithi152 ceph-mon[88640]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:58.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:57 smithi102 ceph-mon[94535]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:58.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:57 smithi102 ceph-mon[98773]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:28:59.758 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:00.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:28:59 smithi152 ceph-mon[88640]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:00.020 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:00.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:28:59 smithi102 ceph-mon[94535]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:00.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:28:59 smithi102 ceph-mon[98773]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:01.247 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:29:00] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-10T03:29:01.874 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:01 smithi102 ceph-mon[94535]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:01.874 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:01 smithi102 ceph-mon[98773]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:02.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:01 smithi152 ceph-mon[88640]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:02.041 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:02.042 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:03.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:02 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3186418550' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:03.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:02 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3186418550' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:03.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:02 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3186418550' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:03 smithi152 ceph-mon[88640]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:04.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:03 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:03 smithi102 ceph-mon[94535]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:04.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:03 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:03 smithi102 ceph-mon[98773]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:04.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:03 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:05.857 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:06.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:05 smithi152 ceph-mon[88640]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:06.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:05 smithi102 ceph-mon[94535]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:06.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:05 smithi102 ceph-mon[98773]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:06.128 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:08.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:07 smithi152 ceph-mon[88640]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:08.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:07 smithi102 ceph-mon[94535]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:08.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:07 smithi102 ceph-mon[98773]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:08.083 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:08.083 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:09.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:08 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/3536134008' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:09.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:08 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/3536134008' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:09.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:08 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/3536134008' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:10.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:09 smithi152 ceph-mon[88640]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:10.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:09 smithi102 ceph-mon[94535]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:10.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:09 smithi102 ceph-mon[98773]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:11.293 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:29:10] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.43.0" 2024-04-10T03:29:11.697 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:11.972 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:12.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:11 smithi152 ceph-mon[88640]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:12.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:11 smithi102 ceph-mon[94535]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:12.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:11 smithi102 ceph-mon[98773]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:13.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:12 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:29:13.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:12 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:29:13.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:12 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-10T03:29:13.792 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[94535]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[98773]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:13.793 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:13 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:14.026 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:14.026 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:14.130 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:13 smithi152 ceph-mon[88640]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:14.131 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:13 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:14.131 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:13 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:14.131 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:13 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:14.131 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:13 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:14.981 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:14 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/564568577' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:14.981 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:14 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/564568577' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:15.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:14 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/564568577' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:15.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:15 smithi152 ceph-mon[88640]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:16.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:15 smithi102 ceph-mon[94535]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:16.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:15 smithi102 ceph-mon[98773]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[98773]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[94535]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:17 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:17 smithi152 ceph-mon[88640]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:18.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:17 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:18.444 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:18.704 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:19.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:29:19.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:18 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:19.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:19.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-10T03:29:19.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-10T03:29:19.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:18 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' 2024-04-10T03:29:20.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:19 smithi102 ceph-mon[94535]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:19 smithi102 ceph-mon[94535]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:19 smithi102 ceph-mon[98773]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:19 smithi102 ceph-mon[98773]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:19 smithi152 ceph-mon[88640]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:19 smithi152 ceph-mon[88640]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:20.673 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:20.674 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:20.977 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:20 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/762761205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:20.978 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:20 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/762761205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:20.978 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:29:20] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:29:21.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:20 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/762761205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:22.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:21 smithi102 ceph-mon[94535]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:22.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:21 smithi102 ceph-mon[98773]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:22.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:21 smithi152 ceph-mon[88640]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:24.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:23 smithi102 ceph-mon[94535]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:24.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:23 smithi102 ceph-mon[98773]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:24.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:23 smithi152 ceph-mon[88640]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:24.398 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:24.673 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:26.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:25 smithi102 ceph-mon[94535]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:26.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:25 smithi102 ceph-mon[98773]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:26.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:25 smithi152 ceph-mon[88640]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:26.698 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:26.699 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:27.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:26 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1003838876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:27.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:26 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1003838876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:27.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:26 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1003838876' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:28.042 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:27 smithi102 ceph-mon[98773]: pgmap v686: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:28.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:27 smithi102 ceph-mon[94535]: pgmap v686: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:28.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:27 smithi152 ceph-mon[88640]: pgmap v686: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:30.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:29 smithi152 ceph-mon[88640]: pgmap v687: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:30.292 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:29 smithi102 ceph-mon[98773]: pgmap v687: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:30.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:29 smithi102 ceph-mon[94535]: pgmap v687: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:30.412 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:30.681 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:31.181 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:29:30] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2024-04-10T03:29:32.052 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:31 smithi102 ceph-mon[94535]: pgmap v688: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:32.052 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:31 smithi102 ceph-mon[98773]: pgmap v688: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:32.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:31 smithi152 ceph-mon[88640]: pgmap v688: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:32.838 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:32.838 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:34.042 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[94535]: pgmap v689: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/1832260705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:34.043 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[94535]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:34.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[98773]: pgmap v689: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:34.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/1832260705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:34.043 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:33 smithi102 ceph-mon[98773]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:33 smithi152 ceph-mon[88640]: pgmap v689: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:34.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:33 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/1832260705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:34.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:33 smithi152 ceph-mon[88640]: from='mgr.25253 172.21.15.102:0/2451907606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-10T03:29:36.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:35 smithi152 ceph-mon[88640]: pgmap v690: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:36.292 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:35 smithi102 ceph-mon[94535]: pgmap v690: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:36.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:35 smithi102 ceph-mon[98773]: pgmap v690: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:36.595 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 -- ceph health --format=json 2024-04-10T03:29:36.861 INFO:teuthology.orchestra.run.smithi102.stderr:Inferring config /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/mon.a/config 2024-04-10T03:29:38.085 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:37 smithi102 ceph-mon[94535]: pgmap v691: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:38.086 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:37 smithi102 ceph-mon[98773]: pgmap v691: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:38.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:37 smithi152 ceph-mon[88640]: pgmap v691: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:38.885 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T03:29:38.885 INFO:teuthology.orchestra.run.smithi102.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-10T03:29:39.636 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-10T03:29:39.873 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e54eb1681bc5438cb0db0f66ee2cca56 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph-c_3a866975a60f6dc859246f6f19fd8ddb4db2cc2e/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-10T03:29:39.876 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-10T03:29:39.885 INFO:tasks.cephadm:Teardown begin 2024-04-10T03:29:39.886 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-10T03:29:39.901 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:39 smithi102 ceph-mon[98773]: pgmap v692: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:39.902 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:39 smithi102 ceph-mon[98773]: from='client.? 172.21.15.102:0/2382914812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:39.902 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:39 smithi102 ceph-mon[94535]: pgmap v692: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:39.902 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:39 smithi102 ceph-mon[94535]: from='client.? 172.21.15.102:0/2382914812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:39.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-10T03:29:39.987 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-10T03:29:39.988 DEBUG:teuthology.orchestra.run.smithi102:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-10T03:29:40.003 DEBUG:teuthology.orchestra.run.smithi152:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-10T03:29:40.025 INFO:tasks.cephadm:Stopping all daemons... 2024-04-10T03:29:40.025 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-10T03:29:40.025 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a 2024-04-10T03:29:40.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:39 smithi152 ceph-mon[88640]: pgmap v692: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-10T03:29:40.263 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:39 smithi152 ceph-mon[88640]: from='client.? 172.21.15.102:0/2382914812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-10T03:29:40.293 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:40 smithi102 systemd[1]: Stopping Ceph mon.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:40.605 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[94530]: 2024-04-10T03:29:40.332+0000 7fa407c59700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:40.606 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a[94530]: 2024-04-10T03:29:40.332+0000 7fa407c59700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-10T03:29:40.606 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE Bus STOPPING 2024-04-10T03:29:40.607 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:29:40.607 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE Bus STOPPED 2024-04-10T03:29:40.607 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE Bus STARTING 2024-04-10T03:29:40.607 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE Serving on http://:::9283 2024-04-10T03:29:40.607 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:40] ENGINE Bus STARTED 2024-04-10T03:29:40.873 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:40 smithi102 podman[150453]: 2024-04-10 03:29:40.60707599 +0000 UTC m=+0.373318671 container died c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-10T03:29:41.139 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: ::ffff:172.21.15.152 - - [10/Apr/2024:03:29:40] "GET /metrics HTTP/1.1" 200 33970 "" "Prometheus/2.43.0" 2024-04-10T03:29:41.534 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:41 smithi102 podman[150453]: 2024-04-10 03:29:41.141509994 +0000 UTC m=+0.907752712 container cleanup c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-10T03:29:41.535 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:41 smithi102 bash[150453]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a 2024-04-10T03:29:41.535 INFO:journalctl@ceph.mon.a.smithi102.stdout:Apr 10 03:29:41 smithi102 podman[150480]: 2024-04-10 03:29:41.351819147 +0000 UTC m=+0.743555227 container remove c3949a498743e3bc261945e5e4021f1aba2e5ca39d170f0e8de67eeccb89a195 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:29:41.549 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.a.service' 2024-04-10T03:29:41.607 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:41.607 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-10T03:29:41.607 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-10T03:29:41.607 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.c 2024-04-10T03:29:41.903 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:41 smithi102 systemd[1]: Stopping Ceph mon.c for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:42.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c[98766]: 2024-04-10T03:29:41.904+0000 7f8857a11700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:42.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c[98766]: 2024-04-10T03:29:41.904+0000 7f8857a11700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-10T03:29:42.293 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:42 smithi102 podman[150591]: 2024-04-10 03:29:42.098880884 +0000 UTC m=+0.310303631 container died ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-10T03:29:42.293 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:41] ENGINE Bus STOPPING 2024-04-10T03:29:42.293 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-10T03:29:42.294 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:41] ENGINE Bus STOPPED 2024-04-10T03:29:42.294 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:41 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:41] ENGINE Bus STARTING 2024-04-10T03:29:42.294 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:42] ENGINE Serving on http://:::9283 2024-04-10T03:29:42.294 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:42 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: [10/Apr/2024:03:29:42] ENGINE Bus STARTED 2024-04-10T03:29:42.746 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:42 smithi102 podman[150591]: 2024-04-10 03:29:42.488419659 +0000 UTC m=+0.699842409 container cleanup ff269d657b5af1ecdec749f17a46a84df0d8a784295c4fe0c08ce7a8d74be1fe (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-10T03:29:42.746 INFO:journalctl@ceph.mon.c.smithi102.stdout:Apr 10 03:29:42 smithi102 bash[150591]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-c 2024-04-10T03:29:42.896 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.c.service' 2024-04-10T03:29:42.933 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:42.933 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-10T03:29:42.934 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-10T03:29:42.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.b 2024-04-10T03:29:43.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:42 smithi152 systemd[1]: Stopping Ceph mon.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:43.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b[88636]: 2024-04-10T03:29:43.266+0000 7f47d176d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:43.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b[88636]: 2024-04-10T03:29:43.266+0000 7f47d176d700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-10T03:29:43.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:43 smithi152 podman[101227]: 2024-04-10 03:29:43.450212467 +0000 UTC m=+0.298100498 container died 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, ceph=True, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-10T03:29:44.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:43 smithi152 podman[101227]: 2024-04-10 03:29:43.894609112 +0000 UTC m=+0.742497151 container cleanup 8c2b763e28e950c113f1a039a307f288bda73636d5ccbcedd2f13933d199b755 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0) 2024-04-10T03:29:44.013 INFO:journalctl@ceph.mon.b.smithi152.stdout:Apr 10 03:29:43 smithi152 bash[101227]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mon-b 2024-04-10T03:29:44.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mon.b.service' 2024-04-10T03:29:44.428 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:44.428 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-10T03:29:44.428 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-10T03:29:44.429 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y 2024-04-10T03:29:44.733 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:44 smithi102 systemd[1]: Stopping Ceph mgr.y for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:45.042 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:29:44.734+0000 7f1bae7da700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:45.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y[84854]: 2024-04-10T03:29:44.734+0000 7f1bae7da700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-10T03:29:45.043 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:44 smithi102 podman[150722]: 2024-04-10 03:29:44.776069405 +0000 UTC m=+0.159235633 container died ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:29:45.518 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:45 smithi102 podman[150722]: 2024-04-10 03:29:45.243444411 +0000 UTC m=+0.626610639 container cleanup ba38a088e643705ab67eceffe665f36f8cae31fc2e5782b625635683ed971444 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, ceph=True, org.label-schema.vendor=CentOS) 2024-04-10T03:29:45.518 INFO:journalctl@ceph.mgr.y.smithi102.stdout:Apr 10 03:29:45 smithi102 bash[150722]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-y 2024-04-10T03:29:45.627 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.y.service' 2024-04-10T03:29:45.663 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:45.664 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-10T03:29:45.664 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-10T03:29:45.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x 2024-04-10T03:29:45.987 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:29:45 smithi152 systemd[1]: Stopping Ceph mgr.x for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:46.262 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:29:45 smithi152 podman[101360]: 2024-04-10 03:29:45.988638721 +0000 UTC m=+0.134721683 container died 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True) 2024-04-10T03:29:46.632 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:29:46 smithi152 podman[101360]: 2024-04-10 03:29:46.365991062 +0000 UTC m=+0.512074023 container cleanup 10f3bdd94f98b0b425b0f878d886269961c170781d5fb28aee7842cf6cfc03bc (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2) 2024-04-10T03:29:46.632 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Apr 10 03:29:46 smithi152 bash[101360]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-mgr-x 2024-04-10T03:29:46.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@mgr.x.service' 2024-04-10T03:29:46.820 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:46.820 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-10T03:29:46.821 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-10T03:29:46.821 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.0 2024-04-10T03:29:47.113 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:46 smithi102 systemd[1]: Stopping Ceph osd.0 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:47.542 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:29:47.113+0000 7f44e648b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:47.543 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:29:47.113+0000 7f44e648b700 -1 osd.0 138 *** Got signal Terminated *** 2024-04-10T03:29:47.543 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:47 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0[107452]: 2024-04-10T03:29:47.113+0000 7f44e648b700 -1 osd.0 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:29:51.013 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:29:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:29:50.590Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:29:51.013 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:29:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:29:50.592Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:29:51.013 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:29:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:29:50.592Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:29:51.013 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:29:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:29:50.594Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:29:51.013 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:29:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:29:50.594Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:29:52.989 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:52 smithi102 podman[150844]: 2024-04-10 03:29:52.710043186 +0000 UTC m=+5.691706488 container died 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-10T03:29:53.292 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:53 smithi102 podman[150844]: 2024-04-10 03:29:53.173099107 +0000 UTC m=+6.154762477 container cleanup 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:29:53.293 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:53 smithi102 bash[150844]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0 2024-04-10T03:29:53.792 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:53 smithi102 podman[150861]: 2024-04-10 03:29:53.491372017 +0000 UTC m=+0.769647948 container remove 0188a8a40b844e73bd07ee6a107458d094f058274bb49c4baf26d1e60410ce60 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:29:54.093 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:53 smithi102 podman[150952]: 2024-04-10 03:29:53.73681967 +0000 UTC m=+0.018358859 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:29:54.093 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:53 smithi102 podman[150952]: 2024-04-10 03:29:53.858879246 +0000 UTC m=+0.140418423 container create 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True) 2024-04-10T03:29:54.382 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:29:54 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:29:54.334Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:29:54.382 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:29:54 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:29:54.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:29:54.383 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:54 smithi102 podman[150952]: 2024-04-10 03:29:54.094722252 +0000 UTC m=+0.376261428 container init 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:29:54.383 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:54 smithi102 podman[150952]: 2024-04-10 03:29:54.098146578 +0000 UTC m=+0.379685765 container start 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-10T03:29:54.383 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:54 smithi102 podman[150952]: 2024-04-10 03:29:54.178098561 +0000 UTC m=+0.459637760 container attach 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403) 2024-04-10T03:29:54.794 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:54 smithi102 podman[150952]: 2024-04-10 03:29:54.382246338 +0000 UTC m=+0.663785518 container died 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-10T03:29:55.046 INFO:journalctl@ceph.osd.0.smithi102.stdout:Apr 10 03:29:54 smithi102 podman[150978]: 2024-04-10 03:29:54.973576285 +0000 UTC m=+0.584444859 container remove 75a4aadd0223f93558e1a4dc0974ec45950f577e8280b797e406339953fffd7d (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, ceph=True, org.label-schema.license=GPLv2) 2024-04-10T03:29:55.051 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.0.service' 2024-04-10T03:29:55.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:29:55.094 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-10T03:29:55.094 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-10T03:29:55.094 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.1 2024-04-10T03:29:55.409 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:29:55 smithi102 systemd[1]: Stopping Ceph osd.1 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:29:55.792 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:29:55 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:29:55.409+0000 7f10bb1fc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:29:55.793 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:29:55 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:29:55.409+0000 7f10bb1fc700 -1 osd.1 138 *** Got signal Terminated *** 2024-04-10T03:29:55.793 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:29:55 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1[116846]: 2024-04-10T03:29:55.409+0000 7f10bb1fc700 -1 osd.1 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:00.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:00.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:00.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:00 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:00.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:01.077 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:00 smithi102 podman[151061]: 2024-04-10 03:30:00.815982805 +0000 UTC m=+5.510688342 container died 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-10T03:30:01.493 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:01 smithi102 podman[151061]: 2024-04-10 03:30:01.260440656 +0000 UTC m=+5.955146192 container cleanup 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697) 2024-04-10T03:30:01.493 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:01 smithi102 bash[151061]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1 2024-04-10T03:30:01.792 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:01 smithi102 podman[151080]: 2024-04-10 03:30:01.495063983 +0000 UTC m=+0.676080409 container remove 2b12e2e39488124a905a3e3ea2f5c42f05c5acee45fd0e50c54074717da76a30 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-10T03:30:02.180 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:01 smithi102 podman[151174]: 2024-04-10 03:30:01.828985169 +0000 UTC m=+0.018230047 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:30:02.180 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:01 smithi102 podman[151174]: 2024-04-10 03:30:01.946283581 +0000 UTC m=+0.135528442 container create 29d29228930da4298744035dd52792dc82df2176fd56f682cee60c381a68b608 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-10T03:30:02.475 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:02 smithi102 podman[151174]: 2024-04-10 03:30:02.181958282 +0000 UTC m=+0.371203144 container init 29d29228930da4298744035dd52792dc82df2176fd56f682cee60c381a68b608 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-04-10T03:30:02.475 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:02 smithi102 podman[151174]: 2024-04-10 03:30:02.185195334 +0000 UTC m=+0.374440209 container start 29d29228930da4298744035dd52792dc82df2176fd56f682cee60c381a68b608 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0) 2024-04-10T03:30:02.475 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:02 smithi102 podman[151174]: 2024-04-10 03:30:02.290344637 +0000 UTC m=+0.479589517 container attach 29d29228930da4298744035dd52792dc82df2176fd56f682cee60c381a68b608 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-10T03:30:02.767 INFO:journalctl@ceph.osd.1.smithi102.stdout:Apr 10 03:30:02 smithi102 podman[151174]: 2024-04-10 03:30:02.477034714 +0000 UTC m=+0.666279566 container died 29d29228930da4298744035dd52792dc82df2176fd56f682cee60c381a68b608 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-1-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-10T03:30:03.053 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.1.service' 2024-04-10T03:30:03.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:03.094 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-10T03:30:03.094 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-10T03:30:03.094 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.2 2024-04-10T03:30:03.292 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:03 smithi102 systemd[1]: Stopping Ceph osd.2 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:03.792 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:30:03.411+0000 7feb2cb75700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:03.793 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:30:03.411+0000 7feb2cb75700 -1 osd.2 138 *** Got signal Terminated *** 2024-04-10T03:30:03.793 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:03 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2[125748]: 2024-04-10T03:30:03.411+0000 7feb2cb75700 -1 osd.2 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:04.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:04.334Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:04.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:04.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:04.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:04 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:04.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:09.129 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:08 smithi102 podman[151280]: 2024-04-10 03:30:08.854795807 +0000 UTC m=+5.538495715 container died 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-10T03:30:09.542 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:09 smithi102 podman[151280]: 2024-04-10 03:30:09.304985975 +0000 UTC m=+5.988685870 container cleanup 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:30:09.543 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:09 smithi102 bash[151280]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2 2024-04-10T03:30:09.919 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:09 smithi102 podman[151299]: 2024-04-10 03:30:09.592816532 +0000 UTC m=+0.735637671 container remove 6784651177ebe244cd74b98ae1241b36688f89f6103bb7bd0e51e1435fad7681 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-10T03:30:10.169 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:10.165Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:10.169 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:10.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:10.170 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:10 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:10.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:10.170 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:09 smithi102 podman[151394]: 2024-04-10 03:30:09.823366984 +0000 UTC m=+0.018065809 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:30:10.170 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:09 smithi102 podman[151394]: 2024-04-10 03:30:09.923867124 +0000 UTC m=+0.118565938 container create 98dec3f82927c252a5cdc916b3bf170c33896c118bcc661a6229c4f3789ce6b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-10T03:30:10.170 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:10 smithi102 podman[151394]: 2024-04-10 03:30:10.168228098 +0000 UTC m=+0.362926912 container init 98dec3f82927c252a5cdc916b3bf170c33896c118bcc661a6229c4f3789ce6b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:30:10.170 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:10 smithi102 podman[151394]: 2024-04-10 03:30:10.171351994 +0000 UTC m=+0.366050817 container start 98dec3f82927c252a5cdc916b3bf170c33896c118bcc661a6229c4f3789ce6b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-10T03:30:10.459 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:10 smithi102 podman[151394]: 2024-04-10 03:30:10.284974488 +0000 UTC m=+0.479673321 container attach 98dec3f82927c252a5cdc916b3bf170c33896c118bcc661a6229c4f3789ce6b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-10T03:30:10.752 INFO:journalctl@ceph.osd.2.smithi102.stdout:Apr 10 03:30:10 smithi102 podman[151394]: 2024-04-10 03:30:10.461678118 +0000 UTC m=+0.656376928 container died 98dec3f82927c252a5cdc916b3bf170c33896c118bcc661a6229c4f3789ce6b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-10T03:30:11.133 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.2.service' 2024-04-10T03:30:11.177 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:11.177 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-10T03:30:11.177 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-10T03:30:11.177 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.3 2024-04-10T03:30:11.402 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:11 smithi102 systemd[1]: Stopping Ceph osd.3 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:11.792 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:30:11.508+0000 7fd069cb3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:11.793 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:30:11.508+0000 7fd069cb3700 -1 osd.3 138 *** Got signal Terminated *** 2024-04-10T03:30:11.793 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:11 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3[130656]: 2024-04-10T03:30:11.508+0000 7fd069cb3700 -1 osd.3 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:14.239 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:13 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:13.886+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:14.240 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:13 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:13.942+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:14.512 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:14 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:14.291+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:14.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:14.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:14.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:14.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:14.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:14 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:14.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:15.180 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:14 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:14.951+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:15.181 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:14 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:14.914+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:15.512 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:15 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:15.332+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:16.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:15 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:15.951+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:16.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:15 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:15.926+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:16.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:16 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:16.335+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:17.149 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:16 smithi102 podman[151501]: 2024-04-10 03:30:16.875375255 +0000 UTC m=+5.472369106 container died 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:30:17.262 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:16 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:16.988+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:17.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:16 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:16.971+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:17.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:17 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:17.061+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:17.542 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:17 smithi102 podman[151501]: 2024-04-10 03:30:17.333142717 +0000 UTC m=+5.930136610 container cleanup 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-10T03:30:17.543 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:17 smithi102 bash[151501]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3 2024-04-10T03:30:17.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:17 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:17.332+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:17.950 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:17 smithi102 podman[151531]: 2024-04-10 03:30:17.601218324 +0000 UTC m=+0.728324850 container remove 7d8b0c0335ac072e80b45ac3ea53936777e343459ce0f82cf4e3be514125f5ba (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-10T03:30:18.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:17 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:17.989+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:18.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:18.096+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:18.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:17 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:17.945+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:18.277 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:17 smithi102 podman[151621]: 2024-04-10 03:30:17.854917522 +0000 UTC m=+0.018283936 image pull 06a486fd9bae0204950caad240e30f29178d356d2d10ad9a03393c127ec8a486 quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f 2024-04-10T03:30:18.278 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:17 smithi102 podman[151621]: 2024-04-10 03:30:17.960130047 +0000 UTC m=+0.123496451 container create caeb18976c1dfa0693f80bd93d9c60096fc6c0f2057f3510e421a1e8143dbbf2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True) 2024-04-10T03:30:18.278 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:18 smithi102 podman[151621]: 2024-04-10 03:30:18.196581085 +0000 UTC m=+0.359947497 container init caeb18976c1dfa0693f80bd93d9c60096fc6c0f2057f3510e421a1e8143dbbf2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-10T03:30:18.278 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:18 smithi102 podman[151621]: 2024-04-10 03:30:18.199972344 +0000 UTC m=+0.363338757 container start caeb18976c1dfa0693f80bd93d9c60096fc6c0f2057f3510e421a1e8143dbbf2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True) 2024-04-10T03:30:18.279 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:18 smithi102 podman[151621]: 2024-04-10 03:30:18.279489886 +0000 UTC m=+0.442856297 container attach caeb18976c1dfa0693f80bd93d9c60096fc6c0f2057f3510e421a1e8143dbbf2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-10T03:30:18.543 INFO:journalctl@ceph.osd.3.smithi102.stdout:Apr 10 03:30:18 smithi102 podman[151621]: 2024-04-10 03:30:18.484721214 +0000 UTC m=+0.648087632 container died caeb18976c1dfa0693f80bd93d9c60096fc6c0f2057f3510e421a1e8143dbbf2 (image=quay.ceph.io/ceph-ci/ceph@sha256:c82bd1098ce1c9f0e0a85f1afac490ec986b1a5d850f766cd3404d990b717a7f, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-3-deactivate, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-04-08-1432-3a86697, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-10T03:30:18.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:18.293+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:19.051 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.3.service' 2024-04-10T03:30:19.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:19.093 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-10T03:30:19.093 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-10T03:30:19.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.4 2024-04-10T03:30:19.262 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:19 smithi152 systemd[1]: Stopping Ceph osd.4 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:19.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:18.944+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:19.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:19.088+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:19.264 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:18 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:18.964+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:19.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:19.271+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:19.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:19.391+0000 7f580d3ec700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:19.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:19.391+0000 7f580d3ec700 -1 osd.4 138 *** Got signal Terminated *** 2024-04-10T03:30:19.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:19.391+0000 7f580d3ec700 -1 osd.4 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:20.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:19.967+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:20.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:20 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:20.127+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:20.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:19 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:19.957+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:20.542 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:20.166Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:20.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:20.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:20.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:20 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:20.167Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:20.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:20 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:20.285+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:21.262 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:21.130+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:21.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:21.007+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:21.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:20 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:20.940+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:21.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:21.265+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:22.262 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:21.969+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:22.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:22 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:22.134+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:22.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:21.993+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:22.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:21 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:21.993+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:22.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:22 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:22.287+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:23.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:22 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:22.960+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:23.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:23.178+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:23.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:23.018+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:23.264 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:23.018+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:23.762 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:23.314+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:23.763 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:23.314+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.160294+0000 front 2024-04-10T03:29:58.160188+0000 (oldest deadline 2024-04-10T03:30:22.860103+0000) 2024-04-10T03:30:24.262 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:23.999+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:24.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:24.147+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:24.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:23.979+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:24.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:23 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:23.979+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:24.686 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:24.310+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:51.159279+0000 front 2024-04-10T03:29:51.159132+0000 (oldest deadline 2024-04-10T03:30:14.059060+0000) 2024-04-10T03:30:24.686 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4[38509]: 2024-04-10T03:30:24.310+0000 7f580b1ed700 -1 osd.4 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.160294+0000 front 2024-04-10T03:29:58.160188+0000 (oldest deadline 2024-04-10T03:30:22.860103+0000) 2024-04-10T03:30:24.794 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:24.336Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:24.794 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:24.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:24.794 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:24 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:24.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:24.975 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:24 smithi152 podman[101501]: 2024-04-10 03:30:24.812893929 +0000 UTC m=+5.517024692 container died 20decec246f6cb4a6452200e06682fc417a758186c633cc9df1c7d1ef20166cd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, ceph=True) 2024-04-10T03:30:25.254 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:25.121+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:25.255 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:25.043+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:25.255 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:25.043+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:25.255 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:25 smithi152 podman[101501]: 2024-04-10 03:30:25.200556421 +0000 UTC m=+5.904687181 container cleanup 20decec246f6cb4a6452200e06682fc417a758186c633cc9df1c7d1ef20166cd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-d3b5060, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-10T03:30:25.256 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:25 smithi152 bash[101501]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4 2024-04-10T03:30:25.256 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:24.976+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:25.256 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:24 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:24.976+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:25.512 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:25 smithi152 podman[101519]: 2024-04-10 03:30:25.435291566 +0000 UTC m=+0.627066845 container remove 20decec246f6cb4a6452200e06682fc417a758186c633cc9df1c7d1ef20166cd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-d3b5060) 2024-04-10T03:30:26.003 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:25 smithi152 podman[101612]: 2024-04-10 03:30:25.752882123 +0000 UTC m=+0.107696549 container create 82e46188a70ef3dfe0dc52a9a1b71d0213e818379ca7414a79314d45ae08361d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4-deactivate, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD) 2024-04-10T03:30:26.004 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:25 smithi152 podman[101612]: 2024-04-10 03:30:25.663456239 +0000 UTC m=+0.018270673 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T03:30:26.004 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:25.958+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:26.004 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:25 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:25.958+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:26.262 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:26.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:26.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:26.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:26.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:26.007+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:26.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:26.007+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:26.264 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:26 smithi152 podman[101612]: 2024-04-10 03:30:26.005475926 +0000 UTC m=+0.360290344 container init 82e46188a70ef3dfe0dc52a9a1b71d0213e818379ca7414a79314d45ae08361d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4-deactivate, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-10T03:30:26.264 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:26 smithi152 podman[101612]: 2024-04-10 03:30:26.009067176 +0000 UTC m=+0.363881594 container start 82e46188a70ef3dfe0dc52a9a1b71d0213e818379ca7414a79314d45ae08361d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD) 2024-04-10T03:30:26.264 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:26 smithi152 podman[101612]: 2024-04-10 03:30:26.130573745 +0000 UTC m=+0.485388179 container attach 82e46188a70ef3dfe0dc52a9a1b71d0213e818379ca7414a79314d45ae08361d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4-deactivate, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-10T03:30:26.633 INFO:journalctl@ceph.osd.4.smithi152.stdout:Apr 10 03:30:26 smithi152 podman[101612]: 2024-04-10 03:30:26.327992427 +0000 UTC m=+0.682806846 container died 82e46188a70ef3dfe0dc52a9a1b71d0213e818379ca7414a79314d45ae08361d (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-04-10T03:30:26.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.4.service' 2024-04-10T03:30:26.993 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:26.994 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-10T03:30:26.994 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-10T03:30:26.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.5 2024-04-10T03:30:27.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:27.058+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:27.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:27.058+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:27.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:26.926+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:27.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:26.926+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:27.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 systemd[1]: Stopping Ceph osd.5 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:27.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:26.979+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:27.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:26 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:26.979+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:27.762 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:27.353+0000 7f83dcbc5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:27.763 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:27.353+0000 7f83dcbc5700 -1 osd.5 138 *** Got signal Terminated *** 2024-04-10T03:30:27.763 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:27.353+0000 7f83dcbc5700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:28.262 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:27.969+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:28.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:27.969+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:28.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:27.934+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:28.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:27 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:27.934+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:28.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:28.105+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:28.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:28.105+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:29.262 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:28.958+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:29.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:28.958+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:29.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:28.940+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:29.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:28 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:28.940+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:29.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:29.094+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:29.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:29.094+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:30.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:29.957+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:30.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:29 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:29.957+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:30.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:30.004+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:30.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:30.004+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:30.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:30.004+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:30.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:30.056+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:30.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:30.056+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:30.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:30 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:30.056+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:30.542 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:30.166Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:30.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:30.167Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:30.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:30 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:30.167Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:31.262 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:31.000+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:31.263 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:31.000+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:31.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:31.026+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:31.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:31.026+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:31.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:31.026+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:31.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:31.054+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:31.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:31.054+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:31.264 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:31 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:31.054+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:32.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:32.098+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:32.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:32.098+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:32.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:32.098+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:32.513 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:32.031+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.245049+0000 front 2024-04-10T03:29:50.245102+0000 (oldest deadline 2024-04-10T03:30:13.144985+0000) 2024-04-10T03:30:32.514 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:32.031+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:54.846014+0000 front 2024-04-10T03:29:54.846075+0000 (oldest deadline 2024-04-10T03:30:21.846356+0000) 2024-04-10T03:30:32.514 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5[44230]: 2024-04-10T03:30:32.031+0000 7f83d99c4700 -1 osd.5 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.047672+0000 front 2024-04-10T03:30:07.047690+0000 (oldest deadline 2024-04-10T03:30:31.147516+0000) 2024-04-10T03:30:32.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:32.074+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:32.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:32.074+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:32.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:32 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:32.074+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:33.119 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:33.091+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:33.120 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:33.091+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:33.120 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:33.091+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:33.120 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:33.049+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:33.120 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:33.049+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:33.121 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:33 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:33.049+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:33.121 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:32 smithi152 podman[101717]: 2024-04-10 03:30:32.844204069 +0000 UTC m=+5.623825004 container died 7eb970dbdd2d52977685f1967eb08f50c9a4b2a2c085ed93714e98a76bd12bfd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:30:33.494 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:33 smithi152 podman[101717]: 2024-04-10 03:30:33.212120357 +0000 UTC m=+5.991741301 container cleanup 7eb970dbdd2d52977685f1967eb08f50c9a4b2a2c085ed93714e98a76bd12bfd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-10T03:30:33.495 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:33 smithi152 bash[101717]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5 2024-04-10T03:30:33.762 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:33 smithi152 podman[101746]: 2024-04-10 03:30:33.496652574 +0000 UTC m=+0.653297299 container remove 7eb970dbdd2d52977685f1967eb08f50c9a4b2a2c085ed93714e98a76bd12bfd (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-10T03:30:34.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:33 smithi152 podman[101842]: 2024-04-10 03:30:33.700692959 +0000 UTC m=+0.017733872 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T03:30:34.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:33 smithi152 podman[101842]: 2024-04-10 03:30:33.822719729 +0000 UTC m=+0.139760686 container create c9938bcc270a60b63c227bb2076ba01b2d5ef7851fda0064acb17a2e442907a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060) 2024-04-10T03:30:34.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:34 smithi152 podman[101842]: 2024-04-10 03:30:34.042136409 +0000 UTC m=+0.359177330 container init c9938bcc270a60b63c227bb2076ba01b2d5ef7851fda0064acb17a2e442907a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-10T03:30:34.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:34 smithi152 podman[101842]: 2024-04-10 03:30:34.045444231 +0000 UTC m=+0.362485144 container start c9938bcc270a60b63c227bb2076ba01b2d5ef7851fda0064acb17a2e442907a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5-deactivate, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-10T03:30:34.341 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:34 smithi152 podman[101842]: 2024-04-10 03:30:34.125177276 +0000 UTC m=+0.442218195 container attach c9938bcc270a60b63c227bb2076ba01b2d5ef7851fda0064acb17a2e442907a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0) 2024-04-10T03:30:34.342 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:34.078+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:34.342 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:34.078+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:34.342 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:34.078+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:34.342 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:34.072+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:34.343 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:34.072+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:34.343 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:34 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:34.072+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:34.634 INFO:journalctl@ceph.osd.5.smithi152.stdout:Apr 10 03:30:34 smithi152 podman[101842]: 2024-04-10 03:30:34.343593468 +0000 UTC m=+0.660634384 container died c9938bcc270a60b63c227bb2076ba01b2d5ef7851fda0064acb17a2e442907a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-5-deactivate, ceph=True, RELEASE=quincy-d3b5060, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409) 2024-04-10T03:30:34.792 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:34.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:34.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:34.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:34.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:34 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:34.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:34.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.5.service' 2024-04-10T03:30:34.979 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:34.979 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-10T03:30:34.980 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-10T03:30:34.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.6 2024-04-10T03:30:35.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:35.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:35.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:35.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:35.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:35.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:35.263 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 systemd[1]: Stopping Ceph osd.6 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:35.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.086+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:35.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.086+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:35.264 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.086+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:35.762 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.314+0000 7f9b29390700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:35.763 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.314+0000 7f9b29390700 -1 osd.6 138 *** Got signal Terminated *** 2024-04-10T03:30:35.763 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:35 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:35.314+0000 7f9b29390700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:36.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:36.046+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:36.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:36.046+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:36.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:36.046+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:36.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:36.146+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:36.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:36.146+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:36.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:36 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:36.146+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:37.164+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:37.164+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:37.164+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:37.084+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:37.084+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:37.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:37 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:37.084+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:38.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:38.112+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:38.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:38.112+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:38.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:38.112+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:38.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:38.126+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:38.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:38.126+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:38.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:38 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:38.126+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:39.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:39.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:39.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:39.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:39.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:39.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:39.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:39.117+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:39.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:39.140+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:39.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:39.140+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:39.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:39.140+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:39.514 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:39 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:39.140+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.281678+0000 front 2024-04-10T03:30:14.281694+0000 (oldest deadline 2024-04-10T03:30:38.381485+0000) 2024-04-10T03:30:40.462 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:40.167Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:40.463 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:40.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:40.463 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:40 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:40.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:40.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:40.137+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:48.677762+0000 front 2024-04-10T03:29:48.677733+0000 (oldest deadline 2024-04-10T03:30:13.377664+0000) 2024-04-10T03:30:40.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:40.137+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:29:58.579364+0000 front 2024-04-10T03:29:58.579345+0000 (oldest deadline 2024-04-10T03:30:24.479331+0000) 2024-04-10T03:30:40.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:40.137+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:07.280825+0000 front 2024-04-10T03:30:07.280812+0000 (oldest deadline 2024-04-10T03:30:28.980697+0000) 2024-04-10T03:30:40.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6[50273]: 2024-04-10T03:30:40.137+0000 7f9b2498c700 -1 osd.6 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.281678+0000 front 2024-04-10T03:30:14.281694+0000 (oldest deadline 2024-04-10T03:30:38.381485+0000) 2024-04-10T03:30:40.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:40.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:40.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:40.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:40.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:40.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:40.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:40 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:40.101+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:41.105 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:40 smithi152 podman[101946]: 2024-04-10 03:30:40.852059446 +0000 UTC m=+5.647446377 container died d97f417e008b37abe69568dff325dc2055fd17f2687ce693846115a5c9dd3aee (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-10T03:30:41.106 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:41 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:41.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:41.106 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:41 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:41.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:41.106 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:41 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:41.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:41.106 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:41 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:41.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:41.497 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:41 smithi152 podman[101946]: 2024-04-10 03:30:41.214491393 +0000 UTC m=+6.009878324 container cleanup d97f417e008b37abe69568dff325dc2055fd17f2687ce693846115a5c9dd3aee (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_CLEAN=True) 2024-04-10T03:30:41.497 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:41 smithi152 bash[101946]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6 2024-04-10T03:30:41.766 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:41 smithi152 podman[101967]: 2024-04-10 03:30:41.499014274 +0000 UTC m=+0.642416958 container remove d97f417e008b37abe69568dff325dc2055fd17f2687ce693846115a5c9dd3aee (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:30:42.178 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:41 smithi152 podman[102061]: 2024-04-10 03:30:41.727285108 +0000 UTC m=+0.018768872 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T03:30:42.178 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:41 smithi152 podman[102061]: 2024-04-10 03:30:41.850001298 +0000 UTC m=+0.141485080 container create 30b8c371dc713e83f89a0145bcef52d8eb6607f27e2880bdb283babc382789ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060) 2024-04-10T03:30:42.178 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:42 smithi152 podman[102061]: 2024-04-10 03:30:42.060636056 +0000 UTC m=+0.352119804 container init 30b8c371dc713e83f89a0145bcef52d8eb6607f27e2880bdb283babc382789ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:42 smithi152 podman[102061]: 2024-04-10 03:30:42.064025261 +0000 UTC m=+0.355509001 container start 30b8c371dc713e83f89a0145bcef52d8eb6607f27e2880bdb283babc382789ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-d3b5060, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:42 smithi152 podman[102061]: 2024-04-10 03:30:42.17732544 +0000 UTC m=+0.468809179 container attach 30b8c371dc713e83f89a0145bcef52d8eb6607f27e2880bdb283babc382789ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, ceph=True) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:42.064+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:42.064+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:42.064+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:42.179 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:42 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:42.064+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:42.513 INFO:journalctl@ceph.osd.6.smithi152.stdout:Apr 10 03:30:42 smithi152 podman[102061]: 2024-04-10 03:30:42.349748221 +0000 UTC m=+0.641231962 container died 30b8c371dc713e83f89a0145bcef52d8eb6607f27e2880bdb283babc382789ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-6-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-d3b5060, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-10T03:30:42.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.6.service' 2024-04-10T03:30:43.011 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:43.011 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-10T03:30:43.012 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-10T03:30:43.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.7 2024-04-10T03:30:43.262 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 systemd[1]: Stopping Ceph osd.7 for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:43.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.111+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:43.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.111+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:43.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.111+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:43.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.111+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:43.762 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.383+0000 7fbeb62e2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-10T03:30:43.763 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.383+0000 7fbeb62e2700 -1 osd.7 138 *** Got signal Terminated *** 2024-04-10T03:30:43.763 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:43 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:43.383+0000 7fbeb62e2700 -1 osd.7 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-10T03:30:44.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:44.141+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:44.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:44.141+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:44.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:44.141+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:44.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:44 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:44.141+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:44.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:44.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:44.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:44.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:44.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:44 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:44.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:45.474 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:45 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:45.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:45.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:45 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:45.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:45.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:45 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:45.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:45.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:45 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:45.097+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:46.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:46.076+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:46.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:46.076+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:46.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:46.076+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:46.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:46 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:46.076+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:47.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:47.084+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:47.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:47.084+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:47.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:47.084+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:47.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:47 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:47.084+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:48.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:48.063+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6806 osd.0 since back 2024-04-10T03:29:50.811861+0000 front 2024-04-10T03:29:50.811740+0000 (oldest deadline 2024-04-10T03:30:16.111953+0000) 2024-04-10T03:30:48.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:48.063+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6814 osd.1 since back 2024-04-10T03:30:00.113450+0000 front 2024-04-10T03:30:00.113432+0000 (oldest deadline 2024-04-10T03:30:26.013426+0000) 2024-04-10T03:30:48.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:48.063+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6822 osd.2 since back 2024-04-10T03:30:06.014163+0000 front 2024-04-10T03:30:06.014149+0000 (oldest deadline 2024-04-10T03:30:29.513951+0000) 2024-04-10T03:30:48.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:48 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7[56148]: 2024-04-10T03:30:48.063+0000 7fbeb08dc700 -1 osd.7 138 heartbeat_check: no reply from 172.21.15.102:6830 osd.3 since back 2024-04-10T03:30:14.115213+0000 front 2024-04-10T03:30:14.115236+0000 (oldest deadline 2024-04-10T03:30:38.215074+0000) 2024-04-10T03:30:49.190 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:48 smithi152 podman[102166]: 2024-04-10 03:30:48.914333235 +0000 UTC m=+5.671393038 container died d60bd93ae519f1e4d812cb16f9662e04b58c91e273c44c77ce7d5fffda2d888f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=quincy-d3b5060, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS) 2024-04-10T03:30:49.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:49 smithi152 podman[102166]: 2024-04-10 03:30:49.29121641 +0000 UTC m=+6.048276215 container cleanup d60bd93ae519f1e4d812cb16f9662e04b58c91e273c44c77ce7d5fffda2d888f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-10T03:30:49.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:49 smithi152 bash[102166]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7 2024-04-10T03:30:49.898 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:49 smithi152 podman[102196]: 2024-04-10 03:30:49.542344376 +0000 UTC m=+0.619380981 container remove d60bd93ae519f1e4d812cb16f9662e04b58c91e273c44c77ce7d5fffda2d888f (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-d3b5060, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, io.buildah.version=1.33.5) 2024-04-10T03:30:50.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:49 smithi152 podman[102289]: 2024-04-10 03:30:49.802673876 +0000 UTC m=+0.018496309 image pull 21de9b7369ad8457bc44c6aca16b9dfec25bf172fb7b6ebd80f97d4df2005241 quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe 2024-04-10T03:30:50.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:49 smithi152 podman[102289]: 2024-04-10 03:30:49.901857951 +0000 UTC m=+0.117680387 container create 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2) 2024-04-10T03:30:50.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:50 smithi152 podman[102289]: 2024-04-10 03:30:50.104195249 +0000 UTC m=+0.320017681 container init 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-d3b5060, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:30:50.263 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:50 smithi152 podman[102289]: 2024-04-10 03:30:50.110731654 +0000 UTC m=+0.326554085 container start 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-d3b5060, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:30:50.542 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:50.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:50.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:50.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-04-10T03:30:50.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:50 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:50.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2024-04-10T03:30:50.738 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:50 smithi152 podman[102289]: 2024-04-10 03:30:50.345776963 +0000 UTC m=+0.561599420 container attach 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-d3b5060, org.label-schema.license=GPLv2) 2024-04-10T03:30:50.739 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:50 smithi152 podman[102289]: 2024-04-10 03:30:50.395686354 +0000 UTC m=+0.611508785 container died 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=quincy-d3b5060, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:30:50.739 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:50.590Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:30:50.739 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:50.592Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:30:50.739 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:50.592Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:30:50.739 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:50.594Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:30:50.740 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:50 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:50.594Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-04-10T03:30:50.999 INFO:journalctl@ceph.osd.7.smithi152.stdout:Apr 10 03:30:50 smithi152 podman[102316]: 2024-04-10 03:30:50.982327195 +0000 UTC m=+0.581633216 container remove 55e0ee3dcff03995c84652232b62665257d4e4b5476cef74b8117a3d561024e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5e334df698f432180b78bc9c46936bfdfbc92bc9a8983463e523f780d23765fe, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-osd-7-deactivate, RELEASE=quincy-d3b5060, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-10T03:30:51.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@osd.7.service' 2024-04-10T03:30:51.046 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:51.046 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-10T03:30:51.047 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-10T03:30:51.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a 2024-04-10T03:30:51.262 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 systemd[1]: Stopping Ceph prometheus.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:51.642 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.376Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-10T03:30:51.643 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a[78373]: ts=2024-04-10T03:30:51.377Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-10T03:30:51.644 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 podman[102395]: 2024-04-10 03:30:51.387200025 +0000 UTC m=+0.109163661 container died a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:52.001 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 podman[102395]: 2024-04-10 03:30:51.751941298 +0000 UTC m=+0.473904925 container cleanup a7900639fc5e743373a9bbd58623829a9847444a0a55905f18f90b5f28c91472 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:52.001 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Apr 10 03:30:51 smithi152 bash[102395]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-prometheus-a 2024-04-10T03:30:52.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@prometheus.a.service' 2024-04-10T03:30:52.193 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-10T03:30:52.194 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-10T03:30:52.194 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 --force --keep-logs 2024-04-10T03:30:53.793 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:53 smithi102 systemd[1]: Stopping Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:54.258 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:53 smithi102 ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a[83825]: ts=2024-04-10T03:30:53.925Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-10T03:30:54.262 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:53 smithi102 podman[152028]: 2024-04-10 03:30:53.976457254 +0000 UTC m=+0.158834831 container died 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:54.542 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 podman[152028]: 2024-04-10 03:30:54.451373058 +0000 UTC m=+0.633750666 container cleanup 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:54.543 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 bash[152028]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a 2024-04-10T03:30:55.043 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 podman[152042]: 2024-04-10 03:30:54.752593006 +0000 UTC m=+0.772588741 container remove 65832d2b0608b4b5138e93e254627e52207c9c1fd869fd17200dde325f38f867 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:55.043 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 podman[152042]: 2024-04-10 03:30:54.847806303 +0000 UTC m=+0.867802039 volume remove 0da420fff6cab3c90be1b37baa9e0e8ba889f9e7127053736066ad67ad3c4883 2024-04-10T03:30:55.043 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-10T03:30:55.043 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 systemd[1]: Stopped Ceph alertmanager.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:30:55.043 INFO:journalctl@ceph.alertmanager.a.smithi102.stdout:Apr 10 03:30:54 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@alertmanager.a.service: Consumed 2.280s CPU time. 2024-04-10T03:30:55.543 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:55 smithi102 systemd[1]: Stopping Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:30:55.543 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:55 smithi102 podman[152150]: 2024-04-10 03:30:55.391523481 +0000 UTC m=+0.114531132 container died 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:56.039 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:55 smithi102 podman[152150]: 2024-04-10 03:30:55.797557792 +0000 UTC m=+0.520565487 container cleanup 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:56.039 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:55 smithi102 bash[152150]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a 2024-04-10T03:30:56.292 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:56 smithi102 podman[152164]: 2024-04-10 03:30:56.0406089 +0000 UTC m=+0.653843710 container remove 773a048f8dafb1f59ba9aa76307b395b84dc50394103fccfa252aca54c777665 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-10T03:30:56.293 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:56 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:30:56.293 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:56 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-10T03:30:56.293 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:56 smithi102 systemd[1]: Stopped Ceph node-exporter.a for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:30:56.293 INFO:journalctl@ceph.node-exporter.a.smithi102.stdout:Apr 10 03:30:56 smithi102 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.a.service: Consumed 3.695s CPU time. 2024-04-10T03:31:06.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 --force --keep-logs 2024-04-10T03:31:08.889 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:08 smithi152 systemd[1]: Stopping Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4... 2024-04-10T03:31:09.170 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:08 smithi152 podman[102841]: 2024-04-10 03:31:08.89025541 +0000 UTC m=+0.111079499 container died 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:31:09.486 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 podman[102841]: 2024-04-10 03:31:09.271018693 +0000 UTC m=+0.491842784 container cleanup 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:31:09.487 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 bash[102841]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b 2024-04-10T03:31:09.762 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 podman[102855]: 2024-04-10 03:31:09.488625242 +0000 UTC m=+0.602665277 container remove 6010c1e762601b432b954e4f32c8935731dcec18a7ea1a6de580945377be430d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-10T03:31:09.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-10T03:31:09.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-10T03:31:09.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 systemd[1]: Stopped Ceph node-exporter.b for e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4. 2024-04-10T03:31:09.763 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Apr 10 03:31:09 smithi152 systemd[1]: ceph-e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4@node-exporter.b.service: Consumed 3.421s CPU time. 2024-04-10T03:31:15.007 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-10T03:31:15.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-10T03:31:15.074 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-10T03:31:15.076 DEBUG:teuthology.misc:Transferring archived files from smithi102:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi102/crash 2024-04-10T03:31:15.077 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash -- . 2024-04-10T03:31:15.126 INFO:teuthology.orchestra.run.smithi102.stderr:tar: /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-10T03:31:15.126 INFO:teuthology.orchestra.run.smithi102.stderr:tar: Error is not recoverable: exiting now 2024-04-10T03:31:15.128 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi152/crash 2024-04-10T03:31:15.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash -- . 2024-04-10T03:31:15.174 INFO:teuthology.orchestra.run.smithi152.stderr:tar: /var/lib/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-10T03:31:15.175 INFO:teuthology.orchestra.run.smithi152.stderr:tar: Error is not recoverable: exiting now 2024-04-10T03:31:15.175 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-10T03:31:15.176 DEBUG:teuthology.orchestra.run.smithi102:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-10T03:31:15.247 INFO:teuthology.orchestra.run.smithi102.stdout:1712718077.3095415 mon.a (mon.0) 501 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-10T03:31:15.247 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-10T03:31:15.247 INFO:tasks.cephadm:Compressing logs... 2024-04-10T03:31:15.247 DEBUG:teuthology.orchestra.run.smithi102:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-10T03:31:15.250 DEBUG:teuthology.orchestra.run.smithi152:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-10T03:31:15.278 INFO:teuthology.orchestra.run.smithi152.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:: No such file or directory 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.6.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi152.adssru.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.6.log: /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.4.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi152.adssru.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.7.log 2024-04-10T03:31:15.279 INFO:teuthology.orchestra.run.smithi152.stderr: 62.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi152.adssru.log.gz 2024-04-10T03:31:15.280 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.5.log 2024-04-10T03:31:15.280 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.x.log 2024-04-10T03:31:15.280 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log 2024-04-10T03:31:15.281 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log 2024-04-10T03:31:15.281 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log 2024-04-10T03:31:15.281 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.b.log 2024-04-10T03:31:15.301 INFO:teuthology.orchestra.run.smithi102.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:: No such file or directory 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.a.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi102.exkvif.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.a.log: /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi102.exkvif.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.1.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr: 81.8% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mds.foo.smithi102.exkvif.log.gz 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.3.log 2024-04-10T03:31:15.302 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.y.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.0.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.2.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log 2024-04-10T03:31:15.303 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.c.log 2024-04-10T03:31:15.305 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log: /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.c.log: 92.8% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log.gz 2024-04-10T03:31:15.389 INFO:teuthology.orchestra.run.smithi102.stderr: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-10T03:31:15.485 INFO:teuthology.orchestra.run.smithi102.stderr: 90.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-10T03:31:15.490 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log: /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log: /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.b.log: 84.2% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-10T03:31:15.499 INFO:teuthology.orchestra.run.smithi152.stderr: 87.5% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.log.gz 2024-04-10T03:31:15.502 INFO:teuthology.orchestra.run.smithi152.stderr: 91.2% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log.gz 2024-04-10T03:31:15.563 INFO:teuthology.orchestra.run.smithi152.stderr: 89.5% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log.gz 2024-04-10T03:31:15.573 INFO:teuthology.orchestra.run.smithi102.stderr: 93.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph.audit.log.gz 2024-04-10T03:31:15.617 INFO:teuthology.orchestra.run.smithi102.stderr: 90.9% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-volume.log.gz 2024-04-10T03:31:15.680 INFO:teuthology.orchestra.run.smithi152.stderr: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-10T03:31:15.758 INFO:teuthology.orchestra.run.smithi152.stderr: 89.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-10T03:31:17.179 INFO:teuthology.orchestra.run.smithi102.stderr: 88.8% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-10T03:31:17.216 INFO:teuthology.orchestra.run.smithi152.stderr: 92.0% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-10T03:31:17.712 INFO:teuthology.orchestra.run.smithi102.stderr: 91.7% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-10T03:31:24.503 INFO:teuthology.orchestra.run.smithi102.stderr: 90.3% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-10T03:31:55.939 INFO:teuthology.orchestra.run.smithi152.stderr: 93.5% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-10T03:31:56.425 INFO:teuthology.orchestra.run.smithi152.stderr: 93.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-10T03:32:01.480 INFO:teuthology.orchestra.run.smithi152.stderr: 93.6% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-10T03:32:15.226 INFO:teuthology.orchestra.run.smithi152.stderr: 93.4% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-10T03:32:15.228 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-04-10T03:32:15.228 INFO:teuthology.orchestra.run.smithi152.stderr:real 0m59.964s 2024-04-10T03:32:15.228 INFO:teuthology.orchestra.run.smithi152.stderr:user 2m52.698s 2024-04-10T03:32:15.228 INFO:teuthology.orchestra.run.smithi152.stderr:sys 0m6.652s 2024-04-10T03:33:03.417 INFO:teuthology.orchestra.run.smithi102.stderr: 93.4% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-10T03:33:17.151 INFO:teuthology.orchestra.run.smithi102.stderr: 93.5% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-10T03:33:29.511 INFO:teuthology.orchestra.run.smithi102.stderr: 93.0% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-10T03:33:37.205 INFO:teuthology.orchestra.run.smithi102.stderr: 93.4% -- replaced with /var/log/ceph/e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-10T03:33:37.207 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-04-10T03:33:37.208 INFO:teuthology.orchestra.run.smithi102.stderr:real 2m21.946s 2024-04-10T03:33:37.208 INFO:teuthology.orchestra.run.smithi102.stderr:user 3m53.227s 2024-04-10T03:33:37.208 INFO:teuthology.orchestra.run.smithi102.stderr:sys 0m9.750s 2024-04-10T03:33:37.208 INFO:tasks.cephadm:Archiving logs... 2024-04-10T03:33:37.208 DEBUG:teuthology.misc:Transferring archived files from smithi102:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi102/log 2024-04-10T03:33:37.210 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-10T03:34:14.057 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi152/log 2024-04-10T03:34:14.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-10T03:34:40.375 INFO:tasks.cephadm:Removing cluster... 2024-04-10T03:34:40.376 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 --force 2024-04-10T03:34:41.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e6f42c6c-f6e5-11ee-b64a-cb9ed24678a4 --force 2024-04-10T03:34:42.079 INFO:tasks.cephadm:Removing cephadm ... 2024-04-10T03:34:42.080 DEBUG:teuthology.orchestra.run.smithi102:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-10T03:34:42.096 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-10T03:34:42.112 INFO:tasks.cephadm:Teardown complete 2024-04-10T03:34:42.113 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-10T03:34:42.125 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-10T03:34:42.125 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-10T03:34:42.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-10T03:34:42.235 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-10T03:34:42.245 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-10T03:34:42.246 DEBUG:teuthology.orchestra.run.smithi102:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-10T03:34:42.248 DEBUG:teuthology.orchestra.run.smithi152:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-10T03:34:42.260 INFO:teuthology.orchestra.run.smithi102.stderr:bash: line 1: ntpq: command not found 2024-04-10T03:34:42.262 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-04-10T03:34:42.266 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-10T03:34:42.266 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-04-10T03:34:42.266 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 54 -128us[ -130us] +/- 81ms 2024-04-10T03:34:42.266 INFO:teuthology.orchestra.run.smithi152.stdout:^* hv02.front.sepia.ceph.com 3 6 377 54 -902us[ -903us] +/- 79ms 2024-04-10T03:34:42.266 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +882us[ +880us] +/- 68ms 2024-04-10T03:34:42.267 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T03:34:42.277 INFO:teuthology.orchestra.run.smithi102.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-10T03:34:42.277 INFO:teuthology.orchestra.run.smithi102.stdout:=============================================================================== 2024-04-10T03:34:42.277 INFO:teuthology.orchestra.run.smithi102.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 53 -140us[ -140us] +/- 81ms 2024-04-10T03:34:42.277 INFO:teuthology.orchestra.run.smithi102.stdout:^* hv02.front.sepia.ceph.com 3 6 377 54 -906us[ -893us] +/- 79ms 2024-04-10T03:34:42.278 INFO:teuthology.orchestra.run.smithi102.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +887us[ +900us] +/- 68ms 2024-04-10T03:34:42.278 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-10T03:34:42.279 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T03:34:42.289 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T03:34:42.289 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T03:34:42.299 DEBUG:teuthology.orchestra.run.smithi102:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T03:34:42.480 DEBUG:teuthology.orchestra.run.smithi152:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T03:34:42.615 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-10T03:34:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T03:34:42.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-10T03:34:42.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T03:34:42.677 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T03:34:42.687 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T03:34:42.697 INFO:teuthology.task.internal:Duration was 2719.608212 seconds 2024-04-10T03:34:42.698 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T03:34:42.708 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T03:34:42.708 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T03:34:42.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T03:34:42.773 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T03:34:42.778 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T03:34:43.099 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T03:34:43.099 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T03:34:43.100 DEBUG:teuthology.orchestra.run.smithi102:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T03:34:43.120 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi152.front.sepia.ceph.com 2024-04-10T03:34:43.120 DEBUG:teuthology.orchestra.run.smithi152:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T03:34:43.147 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T03:34:43.147 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T03:34:43.162 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T03:34:43.216 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T03:34:43.216 DEBUG:teuthology.orchestra.run.smithi102:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T03:34:43.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T03:34:43.709 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T03:34:43.721 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T03:34:43.721 DEBUG:teuthology.orchestra.run.smithi102:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T03:34:43.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T03:34:43.798 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T03:34:43.809 DEBUG:teuthology.orchestra.run.smithi102:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T03:34:43.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T03:34:43.838 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = core 2024-04-10T03:34:43.867 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = core 2024-04-10T03:34:43.892 DEBUG:teuthology.orchestra.run.smithi102:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T03:34:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T03:34:43.920 DEBUG:teuthology.orchestra.run.smithi152:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T03:34:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T03:34:43.946 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T03:34:43.956 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T03:34:43.957 DEBUG:teuthology.misc:Transferring archived files from smithi102:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi102 2024-04-10T03:34:43.957 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T03:34:44.010 DEBUG:teuthology.misc:Transferring archived files from smithi152:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943/remote/smithi152 2024-04-10T03:34:44.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T03:34:44.051 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T03:34:44.051 DEBUG:teuthology.orchestra.run.smithi102:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T03:34:44.054 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T03:34:44.099 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T03:34:44.112 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T03:34:44.112 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T03:34:44.123 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T03:34:44.124 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T03:34:44.127 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T03:34:44.138 INFO:teuthology.orchestra.run.smithi102.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 03:34 /home/ubuntu/cephtest 2024-04-10T03:34:44.153 INFO:teuthology.orchestra.run.smithi152.stdout: 265318 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 03:34 /home/ubuntu/cephtest 2024-04-10T03:34:44.158 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T03:34:44.169 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T03:34:44.193 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T03:34:44.218 DEBUG:teuthology.nuke:shortname: smithi102 2024-04-10T03:34:44.218 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T03:34:44.238 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi102.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 02:40:37.713075', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKUOtJ+0rPgrgh8t0qmo1iUjWt0JrofPmzTh2tK8FTK0nHjSmLQrr5JRPfFEVmtK4CLHgfwnkwRbAIQ258cbKG0='} 2024-04-10T03:34:44.244 DEBUG:teuthology.nuke:shortname: smithi152 2024-04-10T03:34:44.244 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T03:34:44.265 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-04-09_14:58:21-upgrade-wip-yuri4-testing-2024-04-08-1432-distro-default-smithi/7648943', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 02:40:37.714986', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBPIC4a+x5o0awOAOn+29xG+SC+y6GFDZ1FYsJodODlZJFj9O5AwKCkH9eJ7xfk4W0vL2Hfg9QBu+pL6gx5gttg='} 2024-04-10T03:34:44.285 INFO:teuthology.orchestra.console.smithi102:Power off 2024-04-10T03:34:44.285 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T03:34:44.309 DEBUG:teuthology.orchestra.console.smithi102:power off output: Chassis Power Control: Down/Off 2024-04-10T03:34:44.309 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:44.324 INFO:teuthology.orchestra.console.smithi152:Power off 2024-04-10T03:34:44.324 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T03:34:44.333 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is on 2024-04-10T03:34:44.352 DEBUG:teuthology.orchestra.console.smithi152:power off output: Chassis Power Control: Down/Off 2024-04-10T03:34:44.352 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:44.378 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is on 2024-04-10T03:34:48.334 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:48.379 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:48.445 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is on 2024-04-10T03:34:48.490 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is on 2024-04-10T03:34:52.450 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:52.491 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:52.560 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is off 2024-04-10T03:34:52.601 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is on 2024-04-10T03:34:52.662 INFO:teuthology.orchestra.console.smithi102:Power off completed 2024-04-10T03:34:56.603 DEBUG:teuthology.orchestra.console.smithi152:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T03:34:56.715 DEBUG:teuthology.orchestra.console.smithi152:check power output: Chassis Power is off 2024-04-10T03:34:56.816 INFO:teuthology.orchestra.console.smithi152:Power off completed 2024-04-10T03:34:56.920 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2719.6082124710083 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_yuriw@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e54eb1681bc5438cb0db0f66ee2cca56 status: fail success: false 2024-04-10T03:34:56.920 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T03:34:57.029 INFO:teuthology.run:FAIL